Add L0 index and anchor UI updates
This commit is contained in:
@@ -1,5 +1,6 @@
|
||||
// Story Summary - Config
|
||||
// Plugin settings, panel config, and vector config.
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
// Story Summary - Config (v2 简化版)
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
|
||||
import { extension_settings } from "../../../../../../extensions.js";
|
||||
import { EXT_ID } from "../../../core/constants.js";
|
||||
@@ -37,6 +38,7 @@ export function getSummaryPanelConfig() {
|
||||
},
|
||||
vector: null,
|
||||
};
|
||||
|
||||
try {
|
||||
const raw = localStorage.getItem('summary_panel_config');
|
||||
if (!raw) return defaults;
|
||||
@@ -66,15 +68,29 @@ export function saveSummaryPanelConfig(config) {
|
||||
}
|
||||
}
|
||||
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
// 向量配置（简化版 - 只需要 key）
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
|
||||
export function getVectorConfig() {
|
||||
try {
|
||||
const raw = localStorage.getItem('summary_panel_config');
|
||||
if (!raw) return null;
|
||||
const parsed = JSON.parse(raw);
|
||||
const cfg = parsed.vector || null;
|
||||
|
||||
if (cfg && !cfg.textFilterRules) {
|
||||
cfg.textFilterRules = [...DEFAULT_FILTER_RULES];
|
||||
}
|
||||
|
||||
// 简化：统一使用硅基
|
||||
if (cfg) {
|
||||
cfg.engine = 'online';
|
||||
cfg.online = cfg.online || {};
|
||||
cfg.online.provider = 'siliconflow';
|
||||
cfg.online.model = 'BAAI/bge-m3';
|
||||
}
|
||||
|
||||
return cfg;
|
||||
} catch {
|
||||
return null;
|
||||
@@ -90,7 +106,19 @@ export function saveVectorConfig(vectorCfg) {
|
||||
try {
|
||||
const raw = localStorage.getItem('summary_panel_config') || '{}';
|
||||
const parsed = JSON.parse(raw);
|
||||
parsed.vector = vectorCfg;
|
||||
|
||||
// 简化配置
|
||||
parsed.vector = {
|
||||
enabled: vectorCfg?.enabled || false,
|
||||
engine: 'online',
|
||||
online: {
|
||||
provider: 'siliconflow',
|
||||
key: vectorCfg?.online?.key || '',
|
||||
model: 'BAAI/bge-m3',
|
||||
},
|
||||
textFilterRules: vectorCfg?.textFilterRules || DEFAULT_FILTER_RULES,
|
||||
};
|
||||
|
||||
localStorage.setItem('summary_panel_config', JSON.stringify(parsed));
|
||||
CommonSettingStorage.set(SUMMARY_CONFIG_KEY, parsed);
|
||||
} catch (e) {
|
||||
|
||||
@@ -6,7 +6,6 @@ import { chat_metadata } from "../../../../../../../script.js";
|
||||
import { EXT_ID } from "../../../core/constants.js";
|
||||
import { xbLog } from "../../../core/debug-core.js";
|
||||
import { clearEventVectors, deleteEventVectorsByIds } from "../vector/storage/chunk-store.js";
|
||||
import { clearEventTextIndex } from '../vector/retrieval/text-search.js';
|
||||
|
||||
const MODULE_ID = 'summaryStore';
|
||||
const FACTS_LIMIT_PER_SUBJECT = 10;
|
||||
@@ -422,7 +421,6 @@ export async function clearSummaryData(chatId) {
|
||||
await clearEventVectors(chatId);
|
||||
}
|
||||
|
||||
clearEventTextIndex();
|
||||
|
||||
xbLog.info(MODULE_ID, '总结数据已清空');
|
||||
}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// story-summary-ui.js
|
||||
// story-summary-ui.js
|
||||
// iframe 内 UI 逻辑
|
||||
|
||||
(function () {
|
||||
@@ -73,33 +73,6 @@
|
||||
'陌生': 'trend-stranger', '投缘': 'trend-click', '亲密': 'trend-close', '交融': 'trend-merge'
|
||||
};
|
||||
|
||||
const LOCAL_MODELS_INFO = {
|
||||
'bge-small-zh': { desc: '手机/低配适用' },
|
||||
'bge-base-zh': { desc: 'PC 推荐，效果更好' },
|
||||
'e5-small': { desc: '非中文用户' }
|
||||
};
|
||||
|
||||
const ONLINE_PROVIDERS_INFO = {
|
||||
siliconflow: {
|
||||
url: 'https://api.siliconflow.cn',
|
||||
models: ['BAAI/bge-m3', 'BAAI/bge-large-zh-v1.5', 'BAAI/bge-small-zh-v1.5'],
|
||||
hint: '💡 <a href="https://siliconflow.cn" target="_blank">硅基流动</a> 注册即送额度，推荐 BAAI/bge-m3',
|
||||
canFetch: false, urlEditable: false
|
||||
},
|
||||
cohere: {
|
||||
url: 'https://api.cohere.ai',
|
||||
models: ['embed-multilingual-v3.0', 'embed-english-v3.0'],
|
||||
hint: '💡 <a href="https://cohere.com" target="_blank">Cohere</a> 提供免费试用额度',
|
||||
canFetch: false, urlEditable: false
|
||||
},
|
||||
openai: {
|
||||
url: '',
|
||||
models: [],
|
||||
hint: '💡 可用 Hugging Face Space 免费自建<br><button class="btn btn-sm" id="btn-hf-guide" style="margin-top:6px">查看部署指南</button>',
|
||||
canFetch: true, urlEditable: true
|
||||
}
|
||||
};
|
||||
|
||||
const DEFAULT_FILTER_RULES = [
|
||||
{ start: '<think>', end: '</think>' },
|
||||
{ start: '<thinking>', end: '</thinking>' },
|
||||
@@ -119,6 +92,7 @@
|
||||
let summaryData = { keywords: [], events: [], characters: { main: [], relationships: [] }, arcs: [], facts: [] };
|
||||
let localGenerating = false;
|
||||
let vectorGenerating = false;
|
||||
let anchorGenerating = false;
|
||||
let relationChart = null;
|
||||
let relationChartFullscreen = null;
|
||||
let currentEditSection = null;
|
||||
@@ -172,7 +146,7 @@
|
||||
const settingsOpen = $('settings-modal')?.classList.contains('active');
|
||||
if (settingsOpen) config.vector = getVectorConfig();
|
||||
if (!config.vector) {
|
||||
config.vector = { enabled: false, engine: 'online', local: { modelId: 'bge-small-zh' }, online: { provider: 'siliconflow', url: '', key: '', model: '' } };
|
||||
config.vector = { enabled: false, engine: 'online', online: { provider: 'siliconflow', key: '', model: 'BAAI/bge-m3' } };
|
||||
}
|
||||
localStorage.setItem('summary_panel_config', JSON.stringify(config));
|
||||
postMsg('SAVE_PANEL_CONFIG', { config });
|
||||
@@ -186,38 +160,16 @@
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
|
||||
function getVectorConfig() {
|
||||
const safeVal = (id, fallback) => {
|
||||
const el = $(id);
|
||||
if (!el) return fallback;
|
||||
return el.type === 'checkbox' ? el.checked : (el.value?.trim() || fallback);
|
||||
};
|
||||
const safeRadio = (name, fallback) => {
|
||||
const el = document.querySelector(`input[name="${name}"]:checked`);
|
||||
return el?.value || fallback;
|
||||
};
|
||||
const modelSelect = $('vector-model-select');
|
||||
const modelCache = [];
|
||||
if (modelSelect) {
|
||||
for (const opt of modelSelect.options) {
|
||||
if (opt.value) modelCache.push(opt.value);
|
||||
}
|
||||
}
|
||||
const result = {
|
||||
enabled: safeVal('vector-enabled', false),
|
||||
engine: safeRadio('vector-engine', 'online'),
|
||||
local: { modelId: safeVal('local-model-select', 'bge-small-zh') },
|
||||
return {
|
||||
enabled: $('vector-enabled')?.checked || false,
|
||||
engine: 'online',
|
||||
online: {
|
||||
provider: safeVal('online-provider', 'siliconflow'),
|
||||
url: safeVal('vector-api-url', ''),
|
||||
key: safeVal('vector-api-key', ''),
|
||||
model: safeVal('vector-model-select', ''),
|
||||
modelCache
|
||||
}
|
||||
provider: 'siliconflow',
|
||||
key: $('vector-api-key')?.value?.trim() || '',
|
||||
model: 'BAAI/bge-m3',
|
||||
},
|
||||
textFilterRules: collectFilterRules(),
|
||||
};
|
||||
|
||||
// 收集过滤规则
|
||||
result.textFilterRules = collectFilterRules();
|
||||
return result;
|
||||
}
|
||||
|
||||
function loadVectorConfig(cfg) {
|
||||
@@ -225,70 +177,14 @@
|
||||
$('vector-enabled').checked = !!cfg.enabled;
|
||||
$('vector-config-area').classList.toggle('hidden', !cfg.enabled);
|
||||
|
||||
const engine = cfg.engine || 'online';
|
||||
const engineRadio = document.querySelector(`input[name="vector-engine"][value="${engine}"]`);
|
||||
if (engineRadio) engineRadio.checked = true;
|
||||
|
||||
$('local-engine-area').classList.toggle('hidden', engine !== 'local');
|
||||
$('online-engine-area').classList.toggle('hidden', engine !== 'online');
|
||||
|
||||
if (cfg.local?.modelId) {
|
||||
$('local-model-select').value = cfg.local.modelId;
|
||||
updateLocalModelDesc(cfg.local.modelId);
|
||||
}
|
||||
if (cfg.online) {
|
||||
const provider = cfg.online.provider || 'siliconflow';
|
||||
$('online-provider').value = provider;
|
||||
updateOnlineProviderUI(provider);
|
||||
if (cfg.online.url) $('vector-api-url').value = cfg.online.url;
|
||||
if (cfg.online.key) $('vector-api-key').value = cfg.online.key;
|
||||
if (cfg.online.modelCache?.length) {
|
||||
setSelectOptions($('vector-model-select'), cfg.online.modelCache);
|
||||
}
|
||||
if (cfg.online.model) $('vector-model-select').value = cfg.online.model;
|
||||
if (cfg.online?.key) {
|
||||
$('vector-api-key').value = cfg.online.key;
|
||||
}
|
||||
|
||||
// 加载过滤规则
|
||||
renderFilterRules(cfg?.textFilterRules || DEFAULT_FILTER_RULES);
|
||||
}
|
||||
|
||||
function updateLocalModelDesc(modelId) {
|
||||
const info = LOCAL_MODELS_INFO[modelId];
|
||||
$('local-model-desc').textContent = info?.desc || '';
|
||||
}
|
||||
|
||||
function updateOnlineProviderUI(provider) {
|
||||
const info = ONLINE_PROVIDERS_INFO[provider];
|
||||
if (!info) return;
|
||||
|
||||
const urlInput = $('vector-api-url');
|
||||
const urlRow = $('online-url-row');
|
||||
if (info.urlEditable) {
|
||||
urlInput.value = urlInput.value || '';
|
||||
urlInput.disabled = false;
|
||||
urlRow.style.display = '';
|
||||
} else {
|
||||
urlInput.value = info.url;
|
||||
urlInput.disabled = true;
|
||||
urlRow.style.display = 'none';
|
||||
}
|
||||
|
||||
const modelSelect = $('vector-model-select');
|
||||
const fetchBtn = $('btn-fetch-models');
|
||||
if (info.canFetch) {
|
||||
fetchBtn.style.display = '';
|
||||
setHtml(modelSelect, '<option value="">点击拉取或手动输入</option>');
|
||||
} else {
|
||||
fetchBtn.style.display = 'none';
|
||||
setSelectOptions(modelSelect, info.models);
|
||||
}
|
||||
|
||||
setHtml($('provider-hint'), info.hint);
|
||||
const guideBtn = $('btn-hf-guide');
|
||||
if (guideBtn) guideBtn.onclick = e => { e.preventDefault(); openHfGuide(); };
|
||||
}
|
||||
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
// Filter Rules UI
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
|
||||
@@ -352,31 +248,6 @@
|
||||
list.appendChild(div);
|
||||
}
|
||||
|
||||
function updateLocalModelStatus(status, message) {
|
||||
const dot = $('local-model-status').querySelector('.status-dot');
|
||||
const text = $('local-model-status').querySelector('.status-text');
|
||||
dot.className = 'status-dot ' + status;
|
||||
text.textContent = message;
|
||||
|
||||
const btnDownload = $('btn-download-model');
|
||||
const btnCancel = $('btn-cancel-download');
|
||||
const btnDelete = $('btn-delete-model');
|
||||
const progress = $('local-model-progress');
|
||||
|
||||
btnDownload.style.display = (status === 'not_downloaded' || status === 'cached' || status === 'error') ? '' : 'none';
|
||||
btnCancel.style.display = (status === 'downloading') ? '' : 'none';
|
||||
btnDelete.style.display = (status === 'ready' || status === 'cached') ? '' : 'none';
|
||||
progress.classList.toggle('hidden', status !== 'downloading');
|
||||
|
||||
btnDownload.textContent = status === 'cached' ? '加载模型' : status === 'error' ? '重试下载' : '下载模型';
|
||||
}
|
||||
|
||||
function updateLocalModelProgress(percent) {
|
||||
const progress = $('local-model-progress');
|
||||
progress.classList.remove('hidden');
|
||||
progress.querySelector('.progress-inner').style.width = percent + '%';
|
||||
progress.querySelector('.progress-text').textContent = percent + '%';
|
||||
}
|
||||
|
||||
function updateOnlineStatus(status, message) {
|
||||
const dot = $('online-api-status').querySelector('.status-dot');
|
||||
@@ -385,116 +256,129 @@
|
||||
text.textContent = message;
|
||||
}
|
||||
|
||||
function updateOnlineModels(models) {
|
||||
const select = $('vector-model-select');
|
||||
const current = select.value;
|
||||
setSelectOptions(select, models);
|
||||
if (current && models.includes(current)) select.value = current;
|
||||
if (!config.vector) config.vector = { enabled: false, engine: 'online', local: {}, online: {} };
|
||||
if (!config.vector.online) config.vector.online = {};
|
||||
config.vector.online.modelCache = [...models];
|
||||
}
|
||||
|
||||
function updateVectorStats(stats) {
|
||||
$('vector-atom-count').textContent = stats.stateAtoms || 0;
|
||||
$('vector-chunk-count').textContent = stats.chunkCount || 0;
|
||||
$('vector-event-count').textContent = stats.eventVectors || 0;
|
||||
if ($('vector-event-total')) $('vector-event-total').textContent = stats.eventCount || 0;
|
||||
if ($('vector-chunk-count')) $('vector-chunk-count').textContent = stats.chunkCount || 0;
|
||||
if ($('vector-chunk-floors')) $('vector-chunk-floors').textContent = stats.builtFloors || 0;
|
||||
if ($('vector-chunk-total')) $('vector-chunk-total').textContent = stats.totalFloors || 0;
|
||||
if ($('vector-message-count')) $('vector-message-count').textContent = stats.totalMessages || 0;
|
||||
}
|
||||
|
||||
function updateVectorGenProgress(phase, current, total) {
|
||||
const progressId = phase === 'L1' ? 'vector-gen-progress-l1' : 'vector-gen-progress-l2';
|
||||
const progress = $(progressId);
|
||||
const btnGen = $('btn-gen-vectors');
|
||||
const btnCancel = $('btn-cancel-vectors');
|
||||
const btnClear = $('btn-clear-vectors');
|
||||
|
||||
if (current < 0) {
|
||||
progress.classList.add('hidden');
|
||||
const l1Hidden = $('vector-gen-progress-l1').classList.contains('hidden');
|
||||
const l2Hidden = $('vector-gen-progress-l2').classList.contains('hidden');
|
||||
if (l1Hidden && l2Hidden) {
|
||||
btnGen.classList.remove('hidden');
|
||||
btnCancel.classList.add('hidden');
|
||||
btnClear.classList.remove('hidden');
|
||||
vectorGenerating = false;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
vectorGenerating = true;
|
||||
progress.classList.remove('hidden');
|
||||
btnGen.classList.add('hidden');
|
||||
btnCancel.classList.remove('hidden');
|
||||
btnClear.classList.add('hidden');
|
||||
|
||||
const percent = total > 0 ? Math.round(current / total * 100) : 0;
|
||||
progress.querySelector('.progress-inner').style.width = percent + '%';
|
||||
progress.querySelector('.progress-text').textContent = `${current}/${total}`;
|
||||
}
|
||||
|
||||
function showVectorMismatchWarning(show) {
|
||||
$('vector-mismatch-warning').classList.toggle('hidden', !show);
|
||||
}
|
||||
|
||||
function initVectorUI() {
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
// 记忆锚点（L0）UI
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
|
||||
function updateAnchorStats(stats) {
|
||||
const extracted = stats.extracted || 0;
|
||||
const total = stats.total || 0;
|
||||
const pending = stats.pending || 0;
|
||||
const empty = stats.empty || 0;
|
||||
const fail = stats.fail || 0;
|
||||
|
||||
$('anchor-extracted').textContent = extracted;
|
||||
$('anchor-total').textContent = total;
|
||||
$('anchor-pending').textContent = pending;
|
||||
|
||||
const extra = document.getElementById('anchor-extra');
|
||||
if (extra) extra.textContent = `空 ${empty} · 失败 ${fail}`;
|
||||
|
||||
const pendingWrap = $('anchor-pending-wrap');
|
||||
if (pendingWrap) {
|
||||
pendingWrap.classList.toggle('hidden', pending === 0);
|
||||
}
|
||||
|
||||
const emptyWarning = $('vector-empty-l0-warning');
|
||||
if (emptyWarning) {
|
||||
emptyWarning.classList.toggle('hidden', extracted > 0);
|
||||
}
|
||||
}
|
||||
|
||||
function updateAnchorProgress(current, total, message) {
|
||||
const progress = $('anchor-progress');
|
||||
const btnGen = $('btn-anchor-generate');
|
||||
const btnClear = $('btn-anchor-clear');
|
||||
const btnCancel = $('btn-anchor-cancel');
|
||||
|
||||
if (current < 0) {
|
||||
progress.classList.add('hidden');
|
||||
btnGen.classList.remove('hidden');
|
||||
btnClear.classList.remove('hidden');
|
||||
btnCancel.classList.add('hidden');
|
||||
anchorGenerating = false;
|
||||
} else {
|
||||
anchorGenerating = true;
|
||||
progress.classList.remove('hidden');
|
||||
btnGen.classList.add('hidden');
|
||||
btnClear.classList.add('hidden');
|
||||
btnCancel.classList.remove('hidden');
|
||||
|
||||
const percent = total > 0 ? Math.round(current / total * 100) : 0;
|
||||
progress.querySelector('.progress-inner').style.width = percent + '%';
|
||||
progress.querySelector('.progress-text').textContent = message || `${current}/${total}`;
|
||||
}
|
||||
}
|
||||
|
||||
function initAnchorUI() {
|
||||
$('btn-anchor-generate').onclick = () => {
|
||||
if (anchorGenerating) return;
|
||||
postMsg('ANCHOR_GENERATE');
|
||||
};
|
||||
|
||||
$('btn-anchor-clear').onclick = () => {
|
||||
if (confirm('清空所有记忆锚点？（L0 向量也会一并清除）')) {
|
||||
postMsg('ANCHOR_CLEAR');
|
||||
}
|
||||
};
|
||||
|
||||
$('btn-anchor-cancel').onclick = () => {
|
||||
postMsg('ANCHOR_CANCEL');
|
||||
};
|
||||
}
|
||||
|
||||
function initVectorUI() {
|
||||
$('vector-enabled').onchange = e => {
|
||||
$('vector-config-area').classList.toggle('hidden', !e.target.checked);
|
||||
};
|
||||
document.querySelectorAll('input[name="vector-engine"]').forEach(radio => {
|
||||
radio.onchange = e => {
|
||||
const isLocal = e.target.value === 'local';
|
||||
$('local-engine-area').classList.toggle('hidden', !isLocal);
|
||||
$('online-engine-area').classList.toggle('hidden', isLocal);
|
||||
};
|
||||
});
|
||||
$('local-model-select').onchange = e => {
|
||||
updateLocalModelDesc(e.target.value);
|
||||
postMsg('VECTOR_CHECK_LOCAL_MODEL', { modelId: e.target.value });
|
||||
};
|
||||
$('online-provider').onchange = e => updateOnlineProviderUI(e.target.value);
|
||||
$('btn-download-model').onclick = () => postMsg('VECTOR_DOWNLOAD_MODEL', { modelId: $('local-model-select').value });
|
||||
$('btn-cancel-download').onclick = () => postMsg('VECTOR_CANCEL_DOWNLOAD');
|
||||
$('btn-delete-model').onclick = () => {
|
||||
if (confirm('确定删除本地模型缓存？')) postMsg('VECTOR_DELETE_MODEL', { modelId: $('local-model-select').value });
|
||||
};
|
||||
$('btn-fetch-models').onclick = () => {
|
||||
postMsg('VECTOR_FETCH_MODELS', { config: { url: $('vector-api-url').value.trim(), key: $('vector-api-key').value.trim() } });
|
||||
};
|
||||
|
||||
$('btn-test-vector-api').onclick = () => {
|
||||
postMsg('VECTOR_TEST_ONLINE', {
|
||||
provider: $('online-provider').value,
|
||||
config: { url: $('vector-api-url').value.trim(), key: $('vector-api-key').value.trim(), model: $('vector-model-select').value.trim() }
|
||||
provider: 'siliconflow',
|
||||
config: {
|
||||
key: $('vector-api-key').value.trim(),
|
||||
model: 'BAAI/bge-m3',
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
// 过滤规则：添加按钮
|
||||
$('btn-add-filter-rule').onclick = addFilterRule;
|
||||
|
||||
$('btn-gen-vectors').onclick = () => {
|
||||
if (vectorGenerating) return;
|
||||
postMsg('VECTOR_GENERATE', { config: getVectorConfig() });
|
||||
};
|
||||
|
||||
$('btn-clear-vectors').onclick = () => {
|
||||
if (confirm('确定清除当前聊天的向量数据？')) postMsg('VECTOR_CLEAR');
|
||||
if (confirm('?????????')) postMsg('VECTOR_CLEAR');
|
||||
};
|
||||
|
||||
$('btn-cancel-vectors').onclick = () => postMsg('VECTOR_CANCEL_GENERATE');
|
||||
|
||||
// 导入导出
|
||||
$('btn-export-vectors').onclick = () => {
|
||||
$('btn-export-vectors').disabled = true;
|
||||
$('vector-io-status').textContent = '导出中...';
|
||||
$('vector-io-status').textContent = '???...';
|
||||
postMsg('VECTOR_EXPORT');
|
||||
};
|
||||
|
||||
$('btn-import-vectors').onclick = () => {
|
||||
// 让 parent 处理文件选择，避免 iframe 传大文件
|
||||
$('btn-import-vectors').disabled = true;
|
||||
$('vector-io-status').textContent = '导入中...';
|
||||
$('vector-io-status').textContent = '???...';
|
||||
postMsg('VECTOR_IMPORT_PICK');
|
||||
};
|
||||
|
||||
initAnchorUI();
|
||||
postMsg('REQUEST_ANCHOR_STATS');
|
||||
}
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
// Settings Modal
|
||||
@@ -1039,172 +923,48 @@
|
||||
postMsg('FULLSCREEN_CLOSED');
|
||||
}
|
||||
|
||||
function openHfGuide() {
|
||||
$('hf-guide-modal').classList.add('active');
|
||||
renderHfGuideContent();
|
||||
postMsg('FULLSCREEN_OPENED');
|
||||
}
|
||||
function renderArcsEditor(arcs) {
|
||||
const list = arcs?.length ? arcs : [{ name: '', trajectory: '', progress: 0, moments: [] }];
|
||||
const es = $('editor-struct');
|
||||
|
||||
function closeHfGuide() {
|
||||
$('hf-guide-modal').classList.remove('active');
|
||||
postMsg('FULLSCREEN_CLOSED');
|
||||
}
|
||||
|
||||
function renderHfGuideContent() {
|
||||
const body = $('hf-guide-body');
|
||||
if (!body || body.innerHTML.trim()) return;
|
||||
|
||||
setHtml(body, `
|
||||
<div class="hf-guide">
|
||||
<div class="hf-section hf-intro">
|
||||
<div class="hf-intro-text"><strong>免费自建 Embedding 服务</strong>，10 分钟搞定</div>
|
||||
<div class="hf-intro-badges">
|
||||
<span class="hf-badge">🆓 完全免费</span>
|
||||
<span class="hf-badge">⚡ 速度不快</span>
|
||||
<span class="hf-badge">🔐 数据私有</span>
|
||||
</div>
|
||||
</div>
|
||||
<div class="hf-section">
|
||||
<div class="hf-step-header"><span class="hf-step-num">1</span><span class="hf-step-title">创建 Space</span></div>
|
||||
<div class="hf-step-content">
|
||||
<p>访问 <a href="https://huggingface.co/new-space" target="_blank">huggingface.co/new-space</a>，登录后创建：</p>
|
||||
<ul class="hf-checklist">
|
||||
<li>Space name: 随便取（如 <code>my-embedding</code>）</li>
|
||||
<li>SDK: 选 <strong>Docker</strong></li>
|
||||
<li>Hardware: 选 <strong>CPU basic (Free)</strong></li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
<div class="hf-section">
|
||||
<div class="hf-step-header"><span class="hf-step-num">2</span><span class="hf-step-title">上传 3 个文件</span></div>
|
||||
<div class="hf-step-content">
|
||||
<p>在 Space 的 Files 页面，依次创建以下文件：</p>
|
||||
<div class="hf-file">
|
||||
<div class="hf-file-header"><span class="hf-file-icon">📄</span><span class="hf-file-name">requirements.txt</span></div>
|
||||
<pre class="hf-code"><code>fastapi
|
||||
uvicorn
|
||||
sentence-transformers
|
||||
torch</code><button class="copy-btn">复制</button></pre>
|
||||
</div>
|
||||
<div class="hf-file">
|
||||
<div class="hf-file-header"><span class="hf-file-icon">🐍</span><span class="hf-file-name">app.py</span><span class="hf-file-note">主程序</span></div>
|
||||
<pre class="hf-code"><code>import os
|
||||
os.environ["OMP_NUM_THREADS"] = "1"
|
||||
os.environ["MKL_NUM_THREADS"] = "1"
|
||||
os.environ["TOKENIZERS_PARALLELISM"] = "false"
|
||||
|
||||
import torch
|
||||
torch.set_num_threads(1)
|
||||
|
||||
import threading
|
||||
from functools import lru_cache
|
||||
from typing import List, Optional
|
||||
from fastapi import FastAPI, HTTPException, Header
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from pydantic import BaseModel
|
||||
from sentence_transformers import SentenceTransformer
|
||||
|
||||
ACCESS_KEY = os.environ.get("ACCESS_KEY", "")
|
||||
MODEL_ID = "BAAI/bge-m3"
|
||||
|
||||
app = FastAPI()
|
||||
app.add_middleware(CORSMiddleware, allow_origins=["*"], allow_credentials=True, allow_methods=["*"], allow_headers=["*"])
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def get_model():
|
||||
return SentenceTransformer(MODEL_ID, trust_remote_code=True)
|
||||
|
||||
class EmbedRequest(BaseModel):
|
||||
input: List[str]
|
||||
model: Optional[str] = "bge-m3"
|
||||
|
||||
@app.post("/v1/embeddings")
|
||||
async def embed(req: EmbedRequest, authorization: Optional[str] = Header(None)):
|
||||
if ACCESS_KEY and (authorization or "").replace("Bearer ", "").strip() != ACCESS_KEY:
|
||||
raise HTTPException(401, "Unauthorized")
|
||||
embeddings = get_model().encode(req.input, normalize_embeddings=True)
|
||||
return {"data": [{"embedding": e.tolist(), "index": i} for i, e in enumerate(embeddings)]}
|
||||
|
||||
@app.get("/v1/models")
|
||||
async def models():
|
||||
return {"data": [{"id": "bge-m3"}]}
|
||||
|
||||
@app.get("/health")
|
||||
async def health():
|
||||
return {"status": "ok"}
|
||||
|
||||
@app.on_event("startup")
|
||||
async def startup():
|
||||
threading.Thread(target=get_model, daemon=True).start()</code><button class="copy-btn">复制</button></pre>
|
||||
</div>
|
||||
<div class="hf-file">
|
||||
<div class="hf-file-header"><span class="hf-file-icon">🐳</span><span class="hf-file-name">Dockerfile</span></div>
|
||||
<pre class="hf-code"><code>FROM python:3.10-slim
|
||||
WORKDIR /app
|
||||
COPY requirements.txt .
|
||||
RUN pip install --no-cache-dir -r requirements.txt
|
||||
COPY app.py ./
|
||||
RUN python -c "from sentence_transformers import SentenceTransformer; SentenceTransformer('BAAI/bge-m3', trust_remote_code=True)"
|
||||
EXPOSE 7860
|
||||
CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "7860", "--workers", "2"]</code><button class="copy-btn">复制</button></pre>
|
||||
setHtml(es, `
|
||||
<div id="arc-list">
|
||||
${list.map((a, i) => `
|
||||
<div class="struct-item arc-item" data-index="${i}">
|
||||
<div class="struct-row"><input type="text" class="arc-name" placeholder="角色名" value="${h(a.name || '')}"></div>
|
||||
<div class="struct-row"><textarea class="arc-trajectory" rows="2" placeholder="当前状态描述">${h(a.trajectory || '')}</textarea></div>
|
||||
<div class="struct-row">
|
||||
<label style="font-size:.75rem;color:var(--txt3)">进度：<input type="number" class="arc-progress" min="0" max="100" value="${Math.round((a.progress || 0) * 100)}" style="width:64px;display:inline-block"> %</label>
|
||||
</div>
|
||||
<div class="struct-row"><textarea class="arc-moments" rows="3" placeholder="关键时刻，一行一个">${h((a.moments || []).map(m => typeof m === 'string' ? m : m.text).join('\n'))}</textarea></div>
|
||||
<div class="struct-actions"><span>角色弧光 ${i + 1}</span></div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="hf-section">
|
||||
<div class="hf-step-header"><span class="hf-step-num">3</span><span class="hf-step-title">等待构建</span></div>
|
||||
<div class="hf-step-content">
|
||||
<p>上传完成后自动开始构建，约需 <strong>10 分钟</strong>（下载模型）。</p>
|
||||
<p>成功后状态变为 <span class="hf-status-badge">Running</span></p>
|
||||
</div>
|
||||
</div>
|
||||
<div class="hf-section">
|
||||
<div class="hf-step-header"><span class="hf-step-num">4</span><span class="hf-step-title">在插件中配置</span></div>
|
||||
<div class="hf-step-content">
|
||||
<div class="hf-config-table">
|
||||
<div class="hf-config-row"><span class="hf-config-label">服务渠道</span><span class="hf-config-value">OpenAI 兼容</span></div>
|
||||
<div class="hf-config-row"><span class="hf-config-label">API URL</span><span class="hf-config-value"><code>https://用户名-空间名.hf.space</code></span></div>
|
||||
<div class="hf-config-row"><span class="hf-config-label">API Key</span><span class="hf-config-value">随便填</span></div>
|
||||
<div class="hf-config-row"><span class="hf-config-label">模型</span><span class="hf-config-value">点"拉取" → 选 <code>bge-m3</code></span></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="hf-section hf-faq">
|
||||
<div class="hf-faq-title">💡 小提示</div>
|
||||
<ul>
|
||||
<li>URL 格式：<code>https://用户名-空间名.hf.space</code>（减号连接，非斜杠）</li>
|
||||
<li>免费 Space 一段时间无请求会休眠，首次唤醒需等 20-30 秒</li>
|
||||
<li>如需保持常驻，可用 <a href="https://cron-job.org" target="_blank">cron-job.org</a> 每 5 分钟 ping <code>/health</code></li>
|
||||
<li>如需密码，在 Space Settings 设置 <code>ACCESS_KEY</code> 环境变量</li>
|
||||
</ul>
|
||||
</div>
|
||||
`).join('')}
|
||||
</div>
|
||||
<div style="margin-top:8px"><button type="button" class="btn btn-sm" id="arc-add">＋ 新增角色弧光</button></div>
|
||||
`);
|
||||
|
||||
// Add copy button handlers
|
||||
body.querySelectorAll('.copy-btn').forEach(btn => {
|
||||
btn.onclick = async () => {
|
||||
const code = btn.previousElementSibling?.textContent || '';
|
||||
try {
|
||||
await navigator.clipboard.writeText(code);
|
||||
btn.textContent = '已复制';
|
||||
setTimeout(() => btn.textContent = '复制', 1200);
|
||||
} catch {
|
||||
const ta = document.createElement('textarea');
|
||||
ta.value = code;
|
||||
document.body.appendChild(ta);
|
||||
ta.select();
|
||||
document.execCommand('copy');
|
||||
ta.remove();
|
||||
btn.textContent = '已复制';
|
||||
setTimeout(() => btn.textContent = '复制', 1200);
|
||||
}
|
||||
};
|
||||
});
|
||||
}
|
||||
es.querySelectorAll('.arc-item').forEach(addDeleteHandler);
|
||||
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
// Recall Log
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
$('arc-add').onclick = () => {
|
||||
const listEl = $('arc-list');
|
||||
const idx = listEl.querySelectorAll('.arc-item').length;
|
||||
const div = document.createElement('div');
|
||||
div.className = 'struct-item arc-item';
|
||||
div.dataset.index = idx;
|
||||
setHtml(div, `
|
||||
<div class="struct-row"><input type="text" class="arc-name" placeholder="角色名"></div>
|
||||
<div class="struct-row"><textarea class="arc-trajectory" rows="2" placeholder="当前状态描述"></textarea></div>
|
||||
<div class="struct-row">
|
||||
<label style="font-size:.75rem;color:var(--txt3)">进度：<input type="number" class="arc-progress" min="0" max="100" value="0" style="width:64px;display:inline-block"> %</label>
|
||||
</div>
|
||||
<div class="struct-row"><textarea class="arc-moments" rows="3" placeholder="关键时刻，一行一个"></textarea></div>
|
||||
<div class="struct-actions"><span>角色弧光 ${idx + 1}</span></div>
|
||||
`);
|
||||
addDeleteHandler(div);
|
||||
listEl.appendChild(div);
|
||||
};
|
||||
}
|
||||
|
||||
function setRecallLog(text) {
|
||||
lastRecallLogText = text || '';
|
||||
@@ -1357,50 +1117,7 @@ CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "7860", "--workers", "
|
||||
};
|
||||
}
|
||||
|
||||
function renderArcsEditor(arcs) {
|
||||
const list = arcs?.length ? arcs : [{ name: '', trajectory: '', progress: 0, moments: [] }];
|
||||
const es = $('editor-struct');
|
||||
|
||||
setHtml(es, `
|
||||
<div id="arc-list">
|
||||
${list.map((a, i) => `
|
||||
<div class="struct-item arc-item" data-index="${i}">
|
||||
<div class="struct-row"><input type="text" class="arc-name" placeholder="角色名" value="${h(a.name || '')}"></div>
|
||||
<div class="struct-row"><textarea class="arc-trajectory" rows="2" placeholder="当前状态描述">${h(a.trajectory || '')}</textarea></div>
|
||||
<div class="struct-row">
|
||||
<label style="font-size:.75rem;color:var(--txt3)">进度：<input type="number" class="arc-progress" min="0" max="100" value="${Math.round((a.progress || 0) * 100)}" style="width:64px;display:inline-block"> %</label>
|
||||
</div>
|
||||
<div class="struct-row"><textarea class="arc-moments" rows="3" placeholder="关键时刻，一行一个">${h((a.moments || []).map(m => typeof m === 'string' ? m : m.text).join('\n'))}</textarea></div>
|
||||
<div class="struct-actions"><span>角色弧光 ${i + 1}</span></div>
|
||||
</div>
|
||||
`).join('')}
|
||||
</div>
|
||||
<div style="margin-top:8px"><button type="button" class="btn btn-sm" id="arc-add">＋ 新增角色弧光</button></div>
|
||||
`);
|
||||
|
||||
es.querySelectorAll('.arc-item').forEach(addDeleteHandler);
|
||||
|
||||
$('arc-add').onclick = () => {
|
||||
const listEl = $('arc-list');
|
||||
const idx = listEl.querySelectorAll('.arc-item').length;
|
||||
const div = document.createElement('div');
|
||||
div.className = 'struct-item arc-item';
|
||||
div.dataset.index = idx;
|
||||
setHtml(div, `
|
||||
<div class="struct-row"><input type="text" class="arc-name" placeholder="角色名"></div>
|
||||
<div class="struct-row"><textarea class="arc-trajectory" rows="2" placeholder="当前状态描述"></textarea></div>
|
||||
<div class="struct-row">
|
||||
<label style="font-size:.75rem;color:var(--txt3)">进度：<input type="number" class="arc-progress" min="0" max="100" value="0" style="width:64px;display:inline-block"> %</label>
|
||||
</div>
|
||||
<div class="struct-row"><textarea class="arc-moments" rows="3" placeholder="关键时刻，一行一个"></textarea></div>
|
||||
<div class="struct-actions"><span>角色弧光 ${idx + 1}</span></div>
|
||||
`);
|
||||
addDeleteHandler(div);
|
||||
listEl.appendChild(div);
|
||||
};
|
||||
}
|
||||
|
||||
function openEditor(section) {
|
||||
function openEditor(section) {
|
||||
currentEditSection = section;
|
||||
const meta = SECTION_META[section];
|
||||
const es = $('editor-struct');
|
||||
@@ -1615,31 +1332,50 @@ CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "7860", "--workers", "
|
||||
if (d.config) loadVectorConfig(d.config);
|
||||
break;
|
||||
|
||||
case 'VECTOR_LOCAL_MODEL_STATUS':
|
||||
updateLocalModelStatus(d.status, d.message);
|
||||
break;
|
||||
|
||||
case 'VECTOR_LOCAL_MODEL_PROGRESS':
|
||||
updateLocalModelProgress(d.percent);
|
||||
break;
|
||||
|
||||
case 'VECTOR_ONLINE_STATUS':
|
||||
updateOnlineStatus(d.status, d.message);
|
||||
break;
|
||||
|
||||
case 'VECTOR_ONLINE_MODELS':
|
||||
updateOnlineModels(d.models || []);
|
||||
break;
|
||||
|
||||
case 'VECTOR_STATS':
|
||||
updateVectorStats(d.stats);
|
||||
if (d.mismatch !== undefined) showVectorMismatchWarning(d.mismatch);
|
||||
break;
|
||||
|
||||
case 'VECTOR_GEN_PROGRESS':
|
||||
updateVectorGenProgress(d.phase, d.current, d.total);
|
||||
case 'ANCHOR_STATS':
|
||||
updateAnchorStats(d.stats || {});
|
||||
break;
|
||||
|
||||
case 'ANCHOR_GEN_PROGRESS':
|
||||
updateAnchorProgress(d.current, d.total, d.message);
|
||||
break;
|
||||
|
||||
case 'VECTOR_GEN_PROGRESS': {
|
||||
const progress = $('vector-gen-progress');
|
||||
const btnGen = $('btn-gen-vectors');
|
||||
const btnCancel = $('btn-cancel-vectors');
|
||||
const btnClear = $('btn-clear-vectors');
|
||||
|
||||
if (d.current < 0) {
|
||||
progress.classList.add('hidden');
|
||||
btnGen.classList.remove('hidden');
|
||||
btnCancel.classList.add('hidden');
|
||||
btnClear.classList.remove('hidden');
|
||||
vectorGenerating = false;
|
||||
} else {
|
||||
vectorGenerating = true;
|
||||
progress.classList.remove('hidden');
|
||||
btnGen.classList.add('hidden');
|
||||
btnCancel.classList.remove('hidden');
|
||||
btnClear.classList.add('hidden');
|
||||
|
||||
const percent = d.total > 0 ? Math.round(d.current / d.total * 100) : 0;
|
||||
progress.querySelector('.progress-inner').style.width = percent + '%';
|
||||
const displayText = d.message || `${d.phase || ''}: ${d.current}/${d.total}`;
|
||||
progress.querySelector('.progress-text').textContent = displayText;
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
case 'VECTOR_EXPORT_RESULT':
|
||||
$('btn-export-vectors').disabled = false;
|
||||
if (d.success) {
|
||||
@@ -1772,8 +1508,6 @@ CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "7860", "--workers", "
|
||||
$('rel-fs-close').onclick = closeRelationsFullscreen;
|
||||
|
||||
// HF guide
|
||||
$('hf-guide-backdrop').onclick = closeHfGuide;
|
||||
$('hf-guide-close').onclick = closeHfGuide;
|
||||
|
||||
// Character selector
|
||||
$('char-sel-trigger').onclick = e => {
|
||||
|
||||
@@ -2750,3 +2750,126 @@ h1 span {
|
||||
font-size: .8125rem;
|
||||
line-height: 1.8;
|
||||
}
|
||||
|
||||
/* ═══════════════════════════════════════════════════════════════════════════
|
||||
记忆锚点区域（L0）
|
||||
═══════════════════════════════════════════════════════════════════════════ */
|
||||
|
||||
.anchor-section {
|
||||
margin-top: 20px;
|
||||
padding: 16px;
|
||||
background: linear-gradient(135deg, rgba(255, 193, 7, 0.05), rgba(255, 152, 0, 0.05));
|
||||
border: 1px solid rgba(255, 193, 7, 0.3);
|
||||
border-radius: 8px;
|
||||
}
|
||||
|
||||
.anchor-header {
|
||||
margin-bottom: 12px;
|
||||
}
|
||||
|
||||
.anchor-title {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 6px;
|
||||
font-size: 0.875rem;
|
||||
font-weight: 600;
|
||||
color: var(--txt);
|
||||
margin-bottom: 4px;
|
||||
}
|
||||
|
||||
.anchor-icon {
|
||||
font-size: 1rem;
|
||||
}
|
||||
|
||||
.anchor-hint {
|
||||
font-size: 0.75rem;
|
||||
color: var(--txt3);
|
||||
}
|
||||
|
||||
.anchor-stats {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
align-items: center;
|
||||
gap: 8px;
|
||||
font-size: 0.8125rem;
|
||||
color: var(--txt2);
|
||||
margin-bottom: 12px;
|
||||
padding: 8px 12px;
|
||||
background: var(--bg2);
|
||||
border-radius: 6px;
|
||||
}
|
||||
|
||||
.anchor-stat-item {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 4px;
|
||||
}
|
||||
|
||||
.anchor-stat-label {
|
||||
color: var(--txt3);
|
||||
}
|
||||
|
||||
.anchor-stat-value strong {
|
||||
color: var(--hl);
|
||||
font-weight: 600;
|
||||
}
|
||||
|
||||
.anchor-stat-sep {
|
||||
color: var(--txt3);
|
||||
}
|
||||
|
||||
.anchor-stat-pending {
|
||||
color: #f59e0b;
|
||||
font-size: 0.75rem;
|
||||
}
|
||||
|
||||
.anchor-stat-pending strong {
|
||||
font-weight: 600;
|
||||
}
|
||||
|
||||
.anchor-progress {
|
||||
margin-bottom: 12px;
|
||||
}
|
||||
|
||||
.anchor-actions {
|
||||
display: flex;
|
||||
gap: 8px;
|
||||
}
|
||||
|
||||
.anchor-actions .btn {
|
||||
flex: 1;
|
||||
min-width: 0;
|
||||
}
|
||||
|
||||
.vector-empty-warning {
|
||||
font-size: 0.75rem;
|
||||
color: #f59e0b;
|
||||
margin-top: 6px;
|
||||
}
|
||||
|
||||
@media (max-width: 768px) {
|
||||
.anchor-section {
|
||||
padding: 12px;
|
||||
}
|
||||
|
||||
.anchor-stats {
|
||||
flex-direction: column;
|
||||
align-items: flex-start;
|
||||
gap: 4px;
|
||||
}
|
||||
|
||||
.anchor-stat-sep {
|
||||
display: none;
|
||||
}
|
||||
}
|
||||
|
||||
@media (max-width: 480px) {
|
||||
.anchor-title {
|
||||
font-size: 0.8125rem;
|
||||
}
|
||||
|
||||
.anchor-stats {
|
||||
font-size: 0.75rem;
|
||||
padding: 6px 10px;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -83,7 +83,7 @@
|
||||
<!-- Facts -->
|
||||
<section class="card facts">
|
||||
<div class="sec-head">
|
||||
<div class="sec-title">世界状态</div>
|
||||
<div class="sec-title">世界状态</div>
|
||||
<button class="sec-btn" data-section="facts">编辑</button>
|
||||
</div>
|
||||
<div class="facts-list scroll" id="facts-list"></div>
|
||||
@@ -116,6 +116,7 @@
|
||||
<div class="sel-trigger" id="char-sel-trigger">
|
||||
<span id="sel-char-text">选择角色</span>
|
||||
</div>
|
||||
<div class="settings-hint" id="anchor-extra" style="margin-top:-6px"></div>
|
||||
<div class="sel-opts" id="char-sel-opts">
|
||||
<div class="sel-opt" data-value="">暂无角色</div>
|
||||
</div>
|
||||
@@ -346,6 +347,8 @@
|
||||
<div class="tab-pane" id="tab-vector">
|
||||
<div class="settings-section">
|
||||
<div class="settings-section-title">智能记忆（向量检索）</div>
|
||||
|
||||
<!-- 启用开关 -->
|
||||
<div class="settings-checkbox-group">
|
||||
<label class="settings-checkbox">
|
||||
<input type="checkbox" id="vector-enabled">
|
||||
@@ -353,104 +356,34 @@
|
||||
<span class="checkbox-label">启用向量检索</span>
|
||||
</label>
|
||||
</div>
|
||||
|
||||
<div id="vector-config-area" class="hidden">
|
||||
<!-- API Key -->
|
||||
<div class="settings-row" style="margin-top:16px">
|
||||
<div class="settings-field full">
|
||||
<label>Embedding 引擎</label>
|
||||
<div class="engine-selector">
|
||||
<label class="engine-option">
|
||||
<input type="radio" name="vector-engine" value="local">
|
||||
<span>本地模型</span>
|
||||
</label>
|
||||
<label class="engine-option">
|
||||
<input type="radio" name="vector-engine" value="online" checked>
|
||||
<span>在线服务</span>
|
||||
</label>
|
||||
<label>硅基流动 API Key</label>
|
||||
<input type="password" id="vector-api-key" placeholder="sk-xxx">
|
||||
<div class="settings-hint">
|
||||
💡 <a href="https://siliconflow.cn" target="_blank">硅基流动</a>
|
||||
内置使用免费模型（bge-m3、Qwen3-8B），注册认证拿 Key 即可
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Local Engine -->
|
||||
<div id="local-engine-area" class="engine-area hidden">
|
||||
<div class="model-select-row">
|
||||
<select id="local-model-select">
|
||||
<option value="bge-small-zh">中文轻量 (51MB)</option>
|
||||
<option value="bge-base-zh">中文标准 (102MB)</option>
|
||||
<option value="e5-small">多语言 (118MB)</option>
|
||||
</select>
|
||||
</div>
|
||||
<div class="model-desc" id="local-model-desc">手机/低配适用</div>
|
||||
<div class="engine-status-row">
|
||||
<div class="engine-status" id="local-model-status">
|
||||
<span class="status-dot"></span>
|
||||
<span class="status-text">检查中...</span>
|
||||
</div>
|
||||
<div class="engine-actions" id="local-model-actions">
|
||||
<button class="btn btn-sm btn-p" id="btn-download-model">下载</button>
|
||||
<button class="btn btn-sm" id="btn-cancel-download"
|
||||
style="display:none">取消</button>
|
||||
<button class="btn btn-sm btn-del" id="btn-delete-model"
|
||||
style="display:none">删除</button>
|
||||
</div>
|
||||
</div>
|
||||
<div class="engine-progress hidden" id="local-model-progress" style="margin-top: 8px;">
|
||||
<div class="progress-bar">
|
||||
<div class="progress-inner"></div>
|
||||
</div>
|
||||
<span class="progress-text">0%</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Online Engine -->
|
||||
<div id="online-engine-area" class="engine-area">
|
||||
<div class="settings-row">
|
||||
<div class="settings-field full">
|
||||
<label>服务渠道</label>
|
||||
<select id="online-provider">
|
||||
<option value="siliconflow">硅基流动（推荐）</option>
|
||||
<option value="cohere">Cohere</option>
|
||||
<option value="openai">OpenAI 兼容（可自建）</option>
|
||||
</select>
|
||||
</div>
|
||||
</div>
|
||||
<div class="settings-row" id="online-url-row">
|
||||
<div class="settings-field full">
|
||||
<label>API URL</label>
|
||||
<input type="text" id="vector-api-url" placeholder="https://api.siliconflow.cn">
|
||||
</div>
|
||||
</div>
|
||||
<div class="settings-row">
|
||||
<div class="settings-field full">
|
||||
<label>API Key</label>
|
||||
<input type="password" id="vector-api-key" placeholder="sk-xxx">
|
||||
</div>
|
||||
</div>
|
||||
<div class="settings-row">
|
||||
<div class="settings-field full">
|
||||
<label>模型</label>
|
||||
<div style="display:flex;gap:8px">
|
||||
<select id="vector-model-select" style="flex:1">
|
||||
<option value="">请选择模型</option>
|
||||
</select>
|
||||
<button class="btn btn-sm" id="btn-fetch-models"
|
||||
style="display:none">拉取</button>
|
||||
<!-- 测试连接 -->
|
||||
<div class="settings-row">
|
||||
<div class="settings-field full">
|
||||
<div class="engine-status-row">
|
||||
<div class="engine-status" id="online-api-status">
|
||||
<span class="status-dot"></span>
|
||||
<span class="status-text">未测试</span>
|
||||
</div>
|
||||
<button class="btn btn-sm" id="btn-test-vector-api">测试连接</button>
|
||||
</div>
|
||||
</div>
|
||||
<div class="engine-status-row">
|
||||
<div class="engine-status" id="online-api-status">
|
||||
<span class="status-dot"></span>
|
||||
<span class="status-text">未测试</span>
|
||||
</div>
|
||||
<button class="btn btn-sm" id="btn-test-vector-api">测试连接</button>
|
||||
</div>
|
||||
<div class="provider-hint" id="provider-hint">
|
||||
💡 <a href="https://siliconflow.cn" target="_blank">硅基流动</a> 免费、速度快、质量好，推荐
|
||||
BAAI/bge-m3
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- 文本过滤规则 - Redesigned for mobile -->
|
||||
<!-- 文本过滤规则 -->
|
||||
<div class="filter-rules-section">
|
||||
<div class="filter-rules-header">
|
||||
<label>文本过滤规则</label>
|
||||
@@ -463,76 +396,115 @@
|
||||
添加
|
||||
</button>
|
||||
</div>
|
||||
<p class="settings-hint">过滤干扰内容（如思考标签）：遇到「起始」跳过直到「结束」</p>
|
||||
<p class="settings-hint">过滤干扰内容（如思考标签）</p>
|
||||
<div id="filter-rules-list" class="filter-rules-list"></div>
|
||||
</div>
|
||||
|
||||
<!-- Vector Stats -->
|
||||
<!-- ═══════════════════════════════════════════════════════════ -->
|
||||
<!-- 记忆锚点（L0 文本层）-->
|
||||
<!-- ═══════════════════════════════════════════════════════════ -->
|
||||
<div class="anchor-section">
|
||||
<div class="anchor-header">
|
||||
<div class="anchor-title">
|
||||
<span class="anchor-icon">📌</span>
|
||||
<span>记忆锚点</span>
|
||||
</div>
|
||||
<div class="anchor-hint">从对话中提取叙事锚点（情绪、地点、动作、揭示等）</div>
|
||||
</div>
|
||||
|
||||
<div class="anchor-stats" id="anchor-stats">
|
||||
<div class="anchor-stat-item">
|
||||
<span class="anchor-stat-label">已提取楼层:</span>
|
||||
<span class="anchor-stat-value"><strong id="anchor-extracted">0</strong></span>
|
||||
</div>
|
||||
<span class="anchor-stat-sep">/</span>
|
||||
<div class="anchor-stat-item">
|
||||
<span class="anchor-stat-label">总 AI 楼层:</span>
|
||||
<span class="anchor-stat-value"><strong id="anchor-total">0</strong></span>
|
||||
</div>
|
||||
<div class="anchor-stat-pending" id="anchor-pending-wrap">
|
||||
<span>(待提取 <strong id="anchor-pending">0</strong> 楼)</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- 进度条 -->
|
||||
<div class="anchor-progress hidden" id="anchor-progress">
|
||||
<div class="progress-bar">
|
||||
<div class="progress-inner"></div>
|
||||
</div>
|
||||
<span class="progress-text">0/0</span>
|
||||
</div>
|
||||
|
||||
<!-- 操作按钮 -->
|
||||
<div class="anchor-actions" id="anchor-action-row">
|
||||
<button class="btn btn-sm btn-p" id="btn-anchor-generate">生成</button>
|
||||
<button class="btn btn-sm btn-del" id="btn-anchor-clear">清空</button>
|
||||
<button class="btn btn-sm hidden" id="btn-anchor-cancel">取消</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- ═══════════════════════════════════════════════════════════ -->
|
||||
<!-- 当前聊天向量 -->
|
||||
<!-- ═══════════════════════════════════════════════════════════ -->
|
||||
<div class="vector-chat-section">
|
||||
<div class="settings-row">
|
||||
<div class="settings-field full">
|
||||
<label>当前聊天向量</label>
|
||||
<div class="vector-stats" id="vector-stats">
|
||||
<div class="vector-stat-col">
|
||||
<span class="vector-stat-label">事件向量:</span>
|
||||
<span class="vector-stat-label">L0 Atoms:</span>
|
||||
<span class="vector-stat-value"><strong
|
||||
id="vector-event-count">0</strong>/<strong
|
||||
id="vector-event-total">0</strong></span>
|
||||
id="vector-atom-count">0</strong></span>
|
||||
</div>
|
||||
<span class="vector-stat-sep">·</span>
|
||||
<div class="vector-stat-col">
|
||||
<span class="vector-stat-label">Chunks:</span>
|
||||
<span class="vector-stat-label">L1 Chunks:</span>
|
||||
<span class="vector-stat-value"><strong
|
||||
id="vector-chunk-count">0</strong>
|
||||
个（<span id="vector-chunk-floors">0</span>/<span
|
||||
id="vector-chunk-total">0</span> 层）</span>
|
||||
id="vector-chunk-count">0</strong></span>
|
||||
</div>
|
||||
<span class="vector-stat-sep">·</span>
|
||||
<div class="vector-stat-col">
|
||||
<span class="vector-stat-label">消息:</span>
|
||||
<span class="vector-stat-label">L2 Events:</span>
|
||||
<span class="vector-stat-value"><strong
|
||||
id="vector-message-count">0</strong></span>
|
||||
id="vector-event-count">0</strong></span>
|
||||
</div>
|
||||
</div>
|
||||
<div class="vector-mismatch-warning hidden" id="vector-mismatch-warning">
|
||||
⚠ 引擎/模型已变更，需重新生成向量
|
||||
⚠ 需重新生成向量
|
||||
</div>
|
||||
<div class="vector-empty-warning hidden" id="vector-empty-l0-warning">
|
||||
⚠ 记忆锚点为空，建议先生成
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="engine-progress hidden" id="vector-gen-progress-l1">
|
||||
<div style="font-size:.75rem;color:var(--txt3);margin-bottom:4px">L1 片段</div>
|
||||
|
||||
<!-- 进度条 -->
|
||||
<div class="engine-progress hidden" id="vector-gen-progress">
|
||||
<div class="progress-bar">
|
||||
<div class="progress-inner"></div>
|
||||
</div>
|
||||
<span class="progress-text">0/0</span>
|
||||
</div>
|
||||
<div class="engine-progress hidden" id="vector-gen-progress-l2">
|
||||
<div style="font-size:.75rem;color:var(--txt3);margin-bottom:4px">L2 事件</div>
|
||||
<div class="progress-bar">
|
||||
<div class="progress-inner"></div>
|
||||
</div>
|
||||
<span class="progress-text">0/0</span>
|
||||
</div>
|
||||
<div class="settings-hint" id="vector-perf-l1"></div>
|
||||
<div class="settings-hint" id="vector-perf-l2"></div>
|
||||
<div class="settings-btn-row" id="vector-action-row">
|
||||
<button class="btn btn-sm btn-p" id="btn-gen-vectors">生成向量</button>
|
||||
<button class="btn btn-sm btn-del" id="btn-clear-vectors">清除向量</button>
|
||||
<button class="btn btn-sm hidden" id="btn-cancel-vectors">取消</button>
|
||||
</div>
|
||||
<div class="settings-hint" style="margin-top:8px">首次生成向量可能耗时较久，页面短暂卡顿属正常。若本地模型重进酒馆后需重下。
|
||||
<span class="progress-text">0%</span>
|
||||
</div>
|
||||
|
||||
<!-- 向量导入导出 -->
|
||||
<!-- 操作按钮 -->
|
||||
<div class="settings-btn-row" id="vector-action-row">
|
||||
<button class="btn btn-sm btn-p" id="btn-gen-vectors">生成向量</button>
|
||||
<button class="btn btn-sm btn-del" id="btn-clear-vectors">清除</button>
|
||||
<button class="btn btn-sm hidden" id="btn-cancel-vectors">取消</button>
|
||||
</div>
|
||||
|
||||
<div class="settings-hint" style="margin-top:8px">
|
||||
向量化现有 L0/L1/L2 数据（首次可能需要 1-2 分钟）
|
||||
</div>
|
||||
|
||||
<!-- 导入导出 -->
|
||||
<div class="vector-io-section">
|
||||
<div class="settings-row">
|
||||
<div class="settings-field full">
|
||||
<label>向量迁移（跨设备 / 防清缓存）</label>
|
||||
<div class="settings-hint" style="margin-bottom:8px">导出/导入均为 zip 格式，勿解压
|
||||
</div>
|
||||
<div class="settings-btn-row" id="vector-io-row" style="margin-top:8px">
|
||||
<button class="btn btn-sm" id="btn-export-vectors">导出向量</button>
|
||||
<button class="btn btn-sm" id="btn-import-vectors">导入向量</button>
|
||||
<label>向量迁移</label>
|
||||
<div class="settings-btn-row" style="margin-top:8px">
|
||||
<button class="btn btn-sm" id="btn-export-vectors">导出</button>
|
||||
<button class="btn btn-sm" id="btn-import-vectors">导入</button>
|
||||
</div>
|
||||
<div class="settings-hint" id="vector-io-status"></div>
|
||||
</div>
|
||||
@@ -595,8 +567,6 @@
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
<script src="https://cdn.jsdelivr.net/npm/echarts@5/dist/echarts.min.js"></script>
|
||||
<script src="story-summary-ui.js"></script>
|
||||
</body>
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
// Story Summary - 主入口（最终版）
|
||||
//
|
||||
// 稳定目标：
|
||||
@@ -43,18 +43,7 @@ import {
|
||||
import { runSummaryGeneration } from "./generate/generator.js";
|
||||
|
||||
// vector service
|
||||
import {
|
||||
embed,
|
||||
getEngineFingerprint,
|
||||
checkLocalModelStatus,
|
||||
downloadLocalModel,
|
||||
cancelDownload,
|
||||
deleteLocalModelCache,
|
||||
testOnlineService,
|
||||
fetchOnlineModels,
|
||||
isLocalModelLoaded,
|
||||
DEFAULT_LOCAL_MODEL,
|
||||
} from "./vector/utils/embedder.js";
|
||||
import { embed, getEngineFingerprint, testOnlineService } from "./vector/utils/embedder.js";
|
||||
|
||||
import {
|
||||
getMeta,
|
||||
@@ -76,8 +65,20 @@ import {
|
||||
syncOnMessageSwiped,
|
||||
syncOnMessageReceived,
|
||||
} from "./vector/pipeline/chunk-builder.js";
|
||||
import { initStateIntegration, rebuildStateVectors } from "./vector/pipeline/state-integration.js";
|
||||
import { clearStateVectors, getStateAtomsCount, getStateVectorsCount } from "./vector/storage/state-store.js";
|
||||
import {
|
||||
incrementalExtractAtoms,
|
||||
clearAllAtomsAndVectors,
|
||||
cancelL0Extraction,
|
||||
getAnchorStats,
|
||||
initStateIntegration,
|
||||
} from "./vector/pipeline/state-integration.js";
|
||||
import {
|
||||
clearStateVectors,
|
||||
getStateAtoms,
|
||||
getStateAtomsCount,
|
||||
getStateVectorsCount,
|
||||
saveStateVectors,
|
||||
} from "./vector/storage/state-store.js";
|
||||
|
||||
// vector io
|
||||
import { exportVectors, importVectors } from "./vector/storage/vector-io.js";
|
||||
@@ -105,6 +106,7 @@ let eventsRegistered = false;
|
||||
let vectorGenerating = false;
|
||||
let vectorCancelled = false;
|
||||
let vectorAbortController = null;
|
||||
let anchorGenerating = false;
|
||||
|
||||
// ★ 用户消息缓存（解决 GENERATION_STARTED 时 chat 尚未包含用户消息的问题）
|
||||
let lastSentUserMessage = null;
|
||||
@@ -213,6 +215,7 @@ function flushPendingFrameMessages() {
|
||||
if (!iframe?.contentWindow) return;
|
||||
pendingFrameMessages.forEach((p) => postToIframe(iframe, p, "LittleWhiteBox"));
|
||||
pendingFrameMessages = [];
|
||||
sendAnchorStatsToFrame();
|
||||
}
|
||||
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
@@ -260,49 +263,66 @@ async function sendVectorStatsToFrame() {
|
||||
});
|
||||
}
|
||||
|
||||
async function sendLocalModelStatusToFrame(modelId) {
|
||||
if (!modelId) {
|
||||
const cfg = getVectorConfig();
|
||||
modelId = cfg?.local?.modelId || DEFAULT_LOCAL_MODEL;
|
||||
}
|
||||
const status = await checkLocalModelStatus(modelId);
|
||||
postToFrame({
|
||||
type: "VECTOR_LOCAL_MODEL_STATUS",
|
||||
status: status.status,
|
||||
message: status.message,
|
||||
});
|
||||
async function sendAnchorStatsToFrame() {
|
||||
const stats = await getAnchorStats();
|
||||
postToFrame({ type: "ANCHOR_STATS", stats });
|
||||
}
|
||||
|
||||
async function handleDownloadLocalModel(modelId) {
|
||||
try {
|
||||
postToFrame({ type: "VECTOR_LOCAL_MODEL_STATUS", status: "downloading", message: "下载中..." });
|
||||
async function handleAnchorGenerate() {
|
||||
if (anchorGenerating) return;
|
||||
|
||||
await downloadLocalModel(modelId, (percent) => {
|
||||
postToFrame({ type: "VECTOR_LOCAL_MODEL_PROGRESS", percent });
|
||||
const vectorCfg = getVectorConfig();
|
||||
if (!vectorCfg?.enabled) {
|
||||
await executeSlashCommand("/echo severity=warning 请先启用向量检索");
|
||||
return;
|
||||
}
|
||||
|
||||
if (!vectorCfg.online?.key) {
|
||||
postToFrame({ type: "VECTOR_ONLINE_STATUS", status: "error", message: "请配置 API Key" });
|
||||
return;
|
||||
}
|
||||
|
||||
const { chatId, chat } = getContext();
|
||||
if (!chatId || !chat?.length) return;
|
||||
|
||||
anchorGenerating = true;
|
||||
|
||||
postToFrame({ type: "ANCHOR_GEN_PROGRESS", current: 0, total: 1, message: "分析中..." });
|
||||
|
||||
try {
|
||||
await incrementalExtractAtoms(chatId, chat, (message, current, total) => {
|
||||
postToFrame({ type: "ANCHOR_GEN_PROGRESS", current, total, message });
|
||||
});
|
||||
|
||||
postToFrame({ type: "VECTOR_LOCAL_MODEL_STATUS", status: "ready", message: "已就绪" });
|
||||
await sendAnchorStatsToFrame();
|
||||
await sendVectorStatsToFrame();
|
||||
|
||||
xbLog.info(MODULE_ID, "记忆锚点生成完成");
|
||||
} catch (e) {
|
||||
if (e.message === "下载已取消") {
|
||||
postToFrame({ type: "VECTOR_LOCAL_MODEL_STATUS", status: "not_downloaded", message: "已取消" });
|
||||
} else {
|
||||
postToFrame({ type: "VECTOR_LOCAL_MODEL_STATUS", status: "error", message: e.message });
|
||||
}
|
||||
xbLog.error(MODULE_ID, "记忆锚点生成失败", e);
|
||||
await executeSlashCommand(`/echo severity=error 记忆锚点生成失败：${e.message}`);
|
||||
} finally {
|
||||
anchorGenerating = false;
|
||||
postToFrame({ type: "ANCHOR_GEN_PROGRESS", current: -1, total: 0 });
|
||||
}
|
||||
}
|
||||
|
||||
function handleCancelDownload() {
|
||||
cancelDownload();
|
||||
postToFrame({ type: "VECTOR_LOCAL_MODEL_STATUS", status: "not_downloaded", message: "已取消" });
|
||||
async function handleAnchorClear() {
|
||||
const { chatId } = getContext();
|
||||
if (!chatId) return;
|
||||
|
||||
await clearAllAtomsAndVectors(chatId);
|
||||
await sendAnchorStatsToFrame();
|
||||
await sendVectorStatsToFrame();
|
||||
|
||||
await executeSlashCommand("/echo severity=info 记忆锚点已清空");
|
||||
xbLog.info(MODULE_ID, "记忆锚点已清空");
|
||||
}
|
||||
|
||||
async function handleDeleteLocalModel(modelId) {
|
||||
try {
|
||||
await deleteLocalModelCache(modelId);
|
||||
postToFrame({ type: "VECTOR_LOCAL_MODEL_STATUS", status: "not_downloaded", message: "未下载" });
|
||||
} catch (e) {
|
||||
postToFrame({ type: "VECTOR_LOCAL_MODEL_STATUS", status: "error", message: e.message });
|
||||
}
|
||||
function handleAnchorCancel() {
|
||||
cancelL0Extraction();
|
||||
anchorGenerating = false;
|
||||
postToFrame({ type: "ANCHOR_GEN_PROGRESS", current: -1, total: 0 });
|
||||
}
|
||||
|
||||
async function handleTestOnlineService(provider, config) {
|
||||
@@ -319,75 +339,70 @@ async function handleTestOnlineService(provider, config) {
|
||||
}
|
||||
}
|
||||
|
||||
async function handleFetchOnlineModels(config) {
|
||||
try {
|
||||
postToFrame({ type: "VECTOR_ONLINE_STATUS", status: "downloading", message: "拉取中..." });
|
||||
const models = await fetchOnlineModels(config);
|
||||
postToFrame({ type: "VECTOR_ONLINE_MODELS", models });
|
||||
postToFrame({ type: "VECTOR_ONLINE_STATUS", status: "success", message: `找到 ${models.length} 个模型` });
|
||||
} catch (e) {
|
||||
postToFrame({ type: "VECTOR_ONLINE_STATUS", status: "error", message: e.message });
|
||||
}
|
||||
}
|
||||
|
||||
async function handleGenerateVectors(vectorCfg) {
|
||||
if (vectorGenerating) return;
|
||||
|
||||
if (!vectorCfg?.enabled) {
|
||||
postToFrame({ type: "VECTOR_GEN_PROGRESS", phase: "L1", current: -1, total: 0 });
|
||||
postToFrame({ type: "VECTOR_GEN_PROGRESS", phase: "L2", current: -1, total: 0 });
|
||||
postToFrame({ type: "VECTOR_GEN_PROGRESS", phase: "ALL", current: -1, total: 0 });
|
||||
return;
|
||||
}
|
||||
|
||||
const { chatId, chat } = getContext();
|
||||
if (!chatId || !chat?.length) return;
|
||||
|
||||
if (vectorCfg.engine === "online") {
|
||||
if (!vectorCfg.online?.key || !vectorCfg.online?.model) {
|
||||
postToFrame({ type: "VECTOR_ONLINE_STATUS", status: "error", message: "请配置在线服务 API" });
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
if (vectorCfg.engine === "local") {
|
||||
const modelId = vectorCfg.local?.modelId || DEFAULT_LOCAL_MODEL;
|
||||
const status = await checkLocalModelStatus(modelId);
|
||||
if (status.status !== "ready") {
|
||||
postToFrame({ type: "VECTOR_LOCAL_MODEL_STATUS", status: "downloading", message: "正在加载模型..." });
|
||||
try {
|
||||
await downloadLocalModel(modelId, (percent) => {
|
||||
postToFrame({ type: "VECTOR_LOCAL_MODEL_PROGRESS", percent });
|
||||
});
|
||||
postToFrame({ type: "VECTOR_LOCAL_MODEL_STATUS", status: "ready", message: "已就绪" });
|
||||
} catch (e) {
|
||||
xbLog.error(MODULE_ID, "模型加载失败", e);
|
||||
postToFrame({ type: "VECTOR_LOCAL_MODEL_STATUS", status: "error", message: e.message });
|
||||
return;
|
||||
}
|
||||
}
|
||||
if (!vectorCfg.online?.key) {
|
||||
postToFrame({ type: "VECTOR_ONLINE_STATUS", status: "error", message: "请配置 API Key" });
|
||||
return;
|
||||
}
|
||||
|
||||
vectorGenerating = true;
|
||||
vectorCancelled = false;
|
||||
vectorAbortController?.abort?.();
|
||||
vectorAbortController = new AbortController();
|
||||
|
||||
const fingerprint = getEngineFingerprint(vectorCfg);
|
||||
const isLocal = vectorCfg.engine === "local";
|
||||
const batchSize = isLocal ? 5 : 25;
|
||||
const concurrency = isLocal ? 1 : 2;
|
||||
|
||||
// L0 向量重建
|
||||
try {
|
||||
await rebuildStateVectors(chatId, vectorCfg);
|
||||
} catch (e) {
|
||||
xbLog.error(MODULE_ID, "L0 向量重建失败", e);
|
||||
// 不阻塞，继续 L1/L2
|
||||
}
|
||||
const batchSize = 20;
|
||||
|
||||
await clearAllChunks(chatId);
|
||||
await clearEventVectors(chatId);
|
||||
await clearStateVectors(chatId);
|
||||
await updateMeta(chatId, { lastChunkFloor: -1, fingerprint });
|
||||
|
||||
const atoms = getStateAtoms();
|
||||
if (!atoms.length) {
|
||||
postToFrame({ type: "VECTOR_GEN_PROGRESS", phase: "L0", current: 0, total: 0, message: "L0 为空，跳过" });
|
||||
} else {
|
||||
postToFrame({ type: "VECTOR_GEN_PROGRESS", phase: "L0", current: 0, total: atoms.length, message: "L0 向量化..." });
|
||||
|
||||
let l0Completed = 0;
|
||||
for (let i = 0; i < atoms.length; i += batchSize) {
|
||||
if (vectorCancelled) break;
|
||||
|
||||
const batch = atoms.slice(i, i + batchSize);
|
||||
const texts = batch.map(a => a.semantic);
|
||||
try {
|
||||
const vectors = await embed(texts, vectorCfg, { signal: vectorAbortController.signal });
|
||||
const items = batch.map((a, j) => ({
|
||||
atomId: a.atomId,
|
||||
floor: a.floor,
|
||||
vector: vectors[j],
|
||||
}));
|
||||
await saveStateVectors(chatId, items, fingerprint);
|
||||
l0Completed += batch.length;
|
||||
postToFrame({ type: "VECTOR_GEN_PROGRESS", phase: "L0", current: l0Completed, total: atoms.length });
|
||||
} catch (e) {
|
||||
if (e?.name === "AbortError") break;
|
||||
xbLog.error(MODULE_ID, "L0 向量化失败", e);
|
||||
vectorCancelled = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (vectorCancelled) {
|
||||
vectorGenerating = false;
|
||||
return;
|
||||
}
|
||||
|
||||
const allChunks = [];
|
||||
for (let floor = 0; floor < chat.length; floor++) {
|
||||
const chunks = chunkMessage(floor, chat[floor]);
|
||||
@@ -398,148 +413,82 @@ async function handleGenerateVectors(vectorCfg) {
|
||||
await saveChunks(chatId, allChunks);
|
||||
}
|
||||
|
||||
const l1Texts = allChunks.map((c) => c.text);
|
||||
const l1Batches = [];
|
||||
for (let i = 0; i < l1Texts.length; i += batchSize) {
|
||||
l1Batches.push({
|
||||
phase: "L1",
|
||||
texts: l1Texts.slice(i, i + batchSize),
|
||||
startIdx: i,
|
||||
});
|
||||
}
|
||||
|
||||
const l1Texts = allChunks.map(c => c.text);
|
||||
const store = getSummaryStore();
|
||||
const events = store?.json?.events || [];
|
||||
|
||||
// L2: 全量重建（先清空再重建，保持与 L1 一致性）
|
||||
await clearEventVectors(chatId);
|
||||
postToFrame({ type: "VECTOR_GEN_PROGRESS", phase: "L1", current: 0, total: l1Texts.length });
|
||||
|
||||
const l2Pairs = events
|
||||
.map((e) => ({ id: e.id, text: `${e.title || ""} ${e.summary || ""}`.trim() }))
|
||||
.filter((p) => p.text);
|
||||
const l1Vectors = [];
|
||||
let completed = 0;
|
||||
for (let i = 0; i < l1Texts.length; i += batchSize) {
|
||||
if (vectorCancelled) break;
|
||||
|
||||
const l2Batches = [];
|
||||
for (let i = 0; i < l2Pairs.length; i += batchSize) {
|
||||
const batch = l2Pairs.slice(i, i + batchSize);
|
||||
l2Batches.push({
|
||||
phase: "L2",
|
||||
texts: batch.map((p) => p.text),
|
||||
ids: batch.map((p) => p.id),
|
||||
startIdx: i,
|
||||
});
|
||||
}
|
||||
|
||||
const l1Total = allChunks.length;
|
||||
const l2Total = events.length;
|
||||
let l1Completed = 0;
|
||||
let l2Completed = 0;
|
||||
|
||||
postToFrame({ type: "VECTOR_GEN_PROGRESS", phase: "L1", current: 0, total: l1Total });
|
||||
postToFrame({ type: "VECTOR_GEN_PROGRESS", phase: "L2", current: l2Completed, total: l2Total });
|
||||
|
||||
let rateLimitWarned = false;
|
||||
|
||||
const allTasks = [...l1Batches, ...l2Batches];
|
||||
const l1Vectors = new Array(l1Texts.length);
|
||||
const l2VectorItems = [];
|
||||
|
||||
let taskIndex = 0;
|
||||
|
||||
async function worker() {
|
||||
while (taskIndex < allTasks.length) {
|
||||
if (vectorCancelled) break;
|
||||
if (vectorAbortController?.signal?.aborted) break;
|
||||
|
||||
const i = taskIndex++;
|
||||
if (i >= allTasks.length) break;
|
||||
|
||||
const task = allTasks[i];
|
||||
|
||||
try {
|
||||
const vectors = await embed(task.texts, vectorCfg, { signal: vectorAbortController.signal });
|
||||
|
||||
if (task.phase === "L1") {
|
||||
for (let j = 0; j < vectors.length; j++) {
|
||||
l1Vectors[task.startIdx + j] = vectors[j];
|
||||
}
|
||||
l1Completed += task.texts.length;
|
||||
postToFrame({
|
||||
type: "VECTOR_GEN_PROGRESS",
|
||||
phase: "L1",
|
||||
current: Math.min(l1Completed, l1Total),
|
||||
total: l1Total,
|
||||
});
|
||||
} else {
|
||||
for (let j = 0; j < vectors.length; j++) {
|
||||
l2VectorItems.push({ eventId: task.ids[j], vector: vectors[j] });
|
||||
}
|
||||
l2Completed += task.texts.length;
|
||||
postToFrame({
|
||||
type: "VECTOR_GEN_PROGRESS",
|
||||
phase: "L2",
|
||||
current: Math.min(l2Completed, l2Total),
|
||||
total: l2Total,
|
||||
});
|
||||
}
|
||||
} catch (e) {
|
||||
if (e?.name === "AbortError") {
|
||||
xbLog.warn(MODULE_ID, "向量生成已取消（AbortError）");
|
||||
break;
|
||||
}
|
||||
|
||||
xbLog.error(MODULE_ID, `${task.phase} batch 向量化失败`, e);
|
||||
|
||||
const msg = String(e?.message || e);
|
||||
const isRateLike = /429|403|rate|limit|quota/i.test(msg);
|
||||
if (isRateLike && !rateLimitWarned) {
|
||||
rateLimitWarned = true;
|
||||
executeSlashCommand("/echo severity=warning 向量生成遇到速率/配额限制，已进入自动重试。");
|
||||
}
|
||||
|
||||
vectorCancelled = true;
|
||||
vectorAbortController?.abort?.();
|
||||
break;
|
||||
}
|
||||
const batch = l1Texts.slice(i, i + batchSize);
|
||||
try {
|
||||
const vectors = await embed(batch, vectorCfg, { signal: vectorAbortController.signal });
|
||||
l1Vectors.push(...vectors);
|
||||
completed += batch.length;
|
||||
postToFrame({ type: "VECTOR_GEN_PROGRESS", phase: "L1", current: completed, total: l1Texts.length });
|
||||
} catch (e) {
|
||||
if (e?.name === 'AbortError') break;
|
||||
xbLog.error(MODULE_ID, 'L1 向量化失败', e);
|
||||
vectorCancelled = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
await Promise.all(
|
||||
Array(Math.min(concurrency, allTasks.length))
|
||||
.fill(null)
|
||||
.map(() => worker())
|
||||
);
|
||||
|
||||
if (vectorCancelled || vectorAbortController?.signal?.aborted) {
|
||||
postToFrame({ type: "VECTOR_GEN_PROGRESS", phase: "L1", current: -1, total: 0 });
|
||||
postToFrame({ type: "VECTOR_GEN_PROGRESS", phase: "L2", current: -1, total: 0 });
|
||||
vectorGenerating = false;
|
||||
return;
|
||||
}
|
||||
|
||||
if (allChunks.length > 0 && l1Vectors.filter(Boolean).length > 0) {
|
||||
const chunkVectorItems = allChunks
|
||||
.map((chunk, idx) => (l1Vectors[idx] ? { chunkId: chunk.chunkId, vector: l1Vectors[idx] } : null))
|
||||
.filter(Boolean);
|
||||
await saveChunkVectors(chatId, chunkVectorItems, fingerprint);
|
||||
if (!vectorCancelled && l1Vectors.length > 0) {
|
||||
const items = allChunks.map((c, i) => ({ chunkId: c.chunkId, vector: l1Vectors[i] })).filter(x => x.vector);
|
||||
await saveChunkVectors(chatId, items, fingerprint);
|
||||
await updateMeta(chatId, { lastChunkFloor: chat.length - 1 });
|
||||
}
|
||||
|
||||
if (l2VectorItems.length > 0) {
|
||||
await saveEventVectorsToDb(chatId, l2VectorItems, fingerprint);
|
||||
const l2Pairs = events
|
||||
.map(e => ({ id: e.id, text: `${e.title || ''} ${e.summary || ''}`.trim() }))
|
||||
.filter(p => p.text);
|
||||
|
||||
postToFrame({ type: "VECTOR_GEN_PROGRESS", phase: "L2", current: 0, total: l2Pairs.length });
|
||||
let l2Completed = 0;
|
||||
for (let i = 0; i < l2Pairs.length; i += batchSize) {
|
||||
if (vectorCancelled) break;
|
||||
|
||||
const batch = l2Pairs.slice(i, i + batchSize);
|
||||
try {
|
||||
const vectors = await embed(batch.map(p => p.text), vectorCfg, { signal: vectorAbortController.signal });
|
||||
const items = batch.map((p, j) => ({ eventId: p.id, vector: vectors[j] }));
|
||||
await saveEventVectorsToDb(chatId, items, fingerprint);
|
||||
l2Completed += batch.length;
|
||||
postToFrame({ type: "VECTOR_GEN_PROGRESS", phase: "L2", current: l2Completed, total: l2Pairs.length });
|
||||
} catch (e) {
|
||||
if (e?.name === 'AbortError') break;
|
||||
xbLog.error(MODULE_ID, 'L2 向量化失败', e);
|
||||
vectorCancelled = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// 更新 fingerprint（无论之前是否匹配）
|
||||
await updateMeta(chatId, { fingerprint });
|
||||
|
||||
postToFrame({ type: "VECTOR_GEN_PROGRESS", phase: "L1", current: -1, total: 0 });
|
||||
postToFrame({ type: "VECTOR_GEN_PROGRESS", phase: "L2", current: -1, total: 0 });
|
||||
postToFrame({ type: "VECTOR_GEN_PROGRESS", phase: "ALL", current: -1, total: 0 });
|
||||
await sendVectorStatsToFrame();
|
||||
|
||||
vectorGenerating = false;
|
||||
vectorCancelled = false;
|
||||
vectorAbortController = null;
|
||||
|
||||
xbLog.info(MODULE_ID, `向量生成完成: L1=${l1Vectors.filter(Boolean).length}, L2=${l2VectorItems.length}`);
|
||||
xbLog.info(MODULE_ID, `向量生成完成: L0=${atoms.length}, L1=${l1Vectors.length}, L2=${l2Pairs.length}`);
|
||||
}
|
||||
|
||||
async function handleClearVectors() {
|
||||
const { chatId } = getContext();
|
||||
if (!chatId) return;
|
||||
|
||||
await clearEventVectors(chatId);
|
||||
await clearAllChunks(chatId);
|
||||
await clearStateVectors(chatId);
|
||||
await updateMeta(chatId, { lastChunkFloor: -1 });
|
||||
await sendVectorStatsToFrame();
|
||||
await executeSlashCommand('/echo severity=info 向量数据已清除。如需恢复召回功能，请重新点击"生成向量"。');
|
||||
xbLog.info(MODULE_ID, "向量数据已清除");
|
||||
}
|
||||
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
@@ -555,20 +504,10 @@ async function autoVectorizeNewEvents(newEventIds) {
|
||||
const { chatId } = getContext();
|
||||
if (!chatId) return;
|
||||
|
||||
// 本地模型未加载时跳过（不阻塞总结流程）
|
||||
if (vectorCfg.engine === "local") {
|
||||
const modelId = vectorCfg.local?.modelId || DEFAULT_LOCAL_MODEL;
|
||||
if (!isLocalModelLoaded(modelId)) {
|
||||
xbLog.warn(MODULE_ID, "L2 自动向量化跳过：本地模型未加载");
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
const store = getSummaryStore();
|
||||
const events = store?.json?.events || [];
|
||||
const newEventIdSet = new Set(newEventIds);
|
||||
|
||||
// 只取本次新增的 events
|
||||
const newEvents = events.filter((e) => newEventIdSet.has(e.id));
|
||||
if (!newEvents.length) return;
|
||||
|
||||
@@ -580,7 +519,7 @@ async function autoVectorizeNewEvents(newEventIds) {
|
||||
|
||||
try {
|
||||
const fingerprint = getEngineFingerprint(vectorCfg);
|
||||
const batchSize = vectorCfg.engine === "local" ? 5 : 25;
|
||||
const batchSize = 20;
|
||||
|
||||
for (let i = 0; i < pairs.length; i += batchSize) {
|
||||
const batch = pairs.slice(i, i + batchSize);
|
||||
@@ -599,7 +538,6 @@ async function autoVectorizeNewEvents(newEventIds) {
|
||||
await sendVectorStatsToFrame();
|
||||
} catch (e) {
|
||||
xbLog.error(MODULE_ID, "L2 自动向量化失败", e);
|
||||
// 不抛出，不阻塞总结流程
|
||||
}
|
||||
}
|
||||
|
||||
@@ -617,7 +555,6 @@ async function syncEventVectorsOnEdit(oldEvents, newEvents) {
|
||||
const oldIds = new Set((oldEvents || []).map((e) => e.id).filter(Boolean));
|
||||
const newIds = new Set((newEvents || []).map((e) => e.id).filter(Boolean));
|
||||
|
||||
// 找出被删除的 eventIds
|
||||
const deletedIds = [...oldIds].filter((id) => !newIds.has(id));
|
||||
|
||||
if (deletedIds.length > 0) {
|
||||
@@ -635,7 +572,6 @@ async function checkVectorIntegrityAndWarn() {
|
||||
const vectorCfg = getVectorConfig();
|
||||
if (!vectorCfg?.enabled) return;
|
||||
|
||||
// 节流：2分钟内不重复提醒
|
||||
const now = Date.now();
|
||||
if (now - lastVectorWarningAt < VECTOR_WARNING_COOLDOWN_MS) return;
|
||||
|
||||
@@ -646,7 +582,6 @@ async function checkVectorIntegrityAndWarn() {
|
||||
const totalFloors = chat.length;
|
||||
const totalEvents = store?.json?.events?.length || 0;
|
||||
|
||||
// 如果没有总结数据，不需要向量
|
||||
if (totalEvents === 0) return;
|
||||
|
||||
const meta = await getMeta(chatId);
|
||||
@@ -655,18 +590,15 @@ async function checkVectorIntegrityAndWarn() {
|
||||
|
||||
const issues = [];
|
||||
|
||||
// 指纹不匹配
|
||||
if (meta.fingerprint && meta.fingerprint !== fingerprint) {
|
||||
issues.push('向量引擎/模型已变更');
|
||||
}
|
||||
|
||||
// L1 不完整
|
||||
const chunkFloorGap = totalFloors - 1 - (meta.lastChunkFloor ?? -1);
|
||||
if (chunkFloorGap > 0) {
|
||||
issues.push(`${chunkFloorGap} 层片段未向量化`);
|
||||
}
|
||||
|
||||
// L2 不完整
|
||||
const eventVectorGap = totalEvents - stats.eventVectors;
|
||||
if (eventVectorGap > 0) {
|
||||
issues.push(`${eventVectorGap} 个事件未向量化`);
|
||||
@@ -678,19 +610,6 @@ async function checkVectorIntegrityAndWarn() {
|
||||
}
|
||||
}
|
||||
|
||||
async function handleClearVectors() {
|
||||
const { chatId } = getContext();
|
||||
if (!chatId) return;
|
||||
|
||||
await clearEventVectors(chatId);
|
||||
await clearAllChunks(chatId);
|
||||
await clearStateVectors(chatId);
|
||||
await updateMeta(chatId, { lastChunkFloor: -1 });
|
||||
await sendVectorStatsToFrame();
|
||||
await executeSlashCommand('/echo severity=info 向量数据已清除。如需恢复召回功能，请重新点击"生成向量"。');
|
||||
xbLog.info(MODULE_ID, "向量数据已清除");
|
||||
}
|
||||
|
||||
async function maybeAutoBuildChunks() {
|
||||
const cfg = getVectorConfig();
|
||||
if (!cfg?.enabled) return;
|
||||
@@ -701,11 +620,6 @@ async function maybeAutoBuildChunks() {
|
||||
const status = await getChunkBuildStatus();
|
||||
if (status.pending <= 0) return;
|
||||
|
||||
if (cfg.engine === "local") {
|
||||
const modelId = cfg.local?.modelId || DEFAULT_LOCAL_MODEL;
|
||||
if (!isLocalModelLoaded(modelId)) return;
|
||||
}
|
||||
|
||||
try {
|
||||
await buildIncrementalChunks({ vectorConfig: cfg });
|
||||
} catch (e) {
|
||||
@@ -887,10 +801,6 @@ function openPanelForMessage(mesId) {
|
||||
|
||||
sendVectorConfigToFrame();
|
||||
sendVectorStatsToFrame();
|
||||
|
||||
const cfg = getVectorConfig();
|
||||
const modelId = cfg?.local?.modelId || DEFAULT_LOCAL_MODEL;
|
||||
sendLocalModelStatusToFrame(modelId);
|
||||
}
|
||||
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
@@ -1042,10 +952,7 @@ function handleFrameMessage(event) {
|
||||
sendSavedConfigToFrame();
|
||||
sendVectorConfigToFrame();
|
||||
sendVectorStatsToFrame();
|
||||
|
||||
const cfg = getVectorConfig();
|
||||
const modelId = cfg?.local?.modelId || DEFAULT_LOCAL_MODEL;
|
||||
sendLocalModelStatusToFrame(modelId);
|
||||
sendAnchorStatsToFrame();
|
||||
break;
|
||||
}
|
||||
|
||||
@@ -1074,30 +981,10 @@ function handleFrameMessage(event) {
|
||||
postToFrame({ type: "SUMMARY_STATUS", statusText: "已停止" });
|
||||
break;
|
||||
|
||||
case "VECTOR_DOWNLOAD_MODEL":
|
||||
handleDownloadLocalModel(data.modelId);
|
||||
break;
|
||||
|
||||
case "VECTOR_CANCEL_DOWNLOAD":
|
||||
handleCancelDownload();
|
||||
break;
|
||||
|
||||
case "VECTOR_DELETE_MODEL":
|
||||
handleDeleteLocalModel(data.modelId);
|
||||
break;
|
||||
|
||||
case "VECTOR_CHECK_LOCAL_MODEL":
|
||||
sendLocalModelStatusToFrame(data.modelId);
|
||||
break;
|
||||
|
||||
case "VECTOR_TEST_ONLINE":
|
||||
handleTestOnlineService(data.provider, data.config);
|
||||
break;
|
||||
|
||||
case "VECTOR_FETCH_MODELS":
|
||||
handleFetchOnlineModels(data.config);
|
||||
break;
|
||||
|
||||
case "VECTOR_GENERATE":
|
||||
if (data.config) saveVectorConfig(data.config);
|
||||
handleGenerateVectors(data.config);
|
||||
@@ -1109,7 +996,25 @@ function handleFrameMessage(event) {
|
||||
|
||||
case "VECTOR_CANCEL_GENERATE":
|
||||
vectorCancelled = true;
|
||||
cancelL0Extraction();
|
||||
try { vectorAbortController?.abort?.(); } catch {}
|
||||
postToFrame({ type: "VECTOR_GEN_PROGRESS", phase: "ALL", current: -1, total: 0 });
|
||||
break;
|
||||
|
||||
case "ANCHOR_GENERATE":
|
||||
handleAnchorGenerate();
|
||||
break;
|
||||
|
||||
case "ANCHOR_CLEAR":
|
||||
handleAnchorClear();
|
||||
break;
|
||||
|
||||
case "ANCHOR_CANCEL":
|
||||
handleAnchorCancel();
|
||||
break;
|
||||
|
||||
case "REQUEST_ANCHOR_STATS":
|
||||
sendAnchorStatsToFrame();
|
||||
break;
|
||||
|
||||
case "VECTOR_EXPORT":
|
||||
|
||||
251
modules/story-summary/vector/llm/atom-extraction.js
Normal file
251
modules/story-summary/vector/llm/atom-extraction.js
Normal file
@@ -0,0 +1,251 @@
|
||||
// ============================================================================
|
||||
// atom-extraction.js - 30并发 + 首批错开 + 取消支持 + 进度回调
|
||||
// ============================================================================
|
||||
|
||||
import { callLLM, parseJson } from './llm-service.js';
|
||||
import { xbLog } from '../../../../core/debug-core.js';
|
||||
import { filterText } from '../utils/text-filter.js';
|
||||
|
||||
const MODULE_ID = 'atom-extraction';
|
||||
|
||||
const CONCURRENCY = 10;
|
||||
const RETRY_COUNT = 2;
|
||||
const RETRY_DELAY = 500;
|
||||
const DEFAULT_TIMEOUT = 20000;
|
||||
const STAGGER_DELAY = 80; // 首批错开延迟（ms）
|
||||
|
||||
let batchCancelled = false;
|
||||
|
||||
export function cancelBatchExtraction() {
|
||||
batchCancelled = true;
|
||||
}
|
||||
|
||||
export function isBatchCancelled() {
|
||||
return batchCancelled;
|
||||
}
|
||||
|
||||
const SYSTEM_PROMPT = `你是叙事锚点提取器。从一轮对话（用户发言+角色回复）中提取4-8个关键锚点。
|
||||
|
||||
只输出JSON：
|
||||
{"atoms":[{"t":"类型","s":"主体","v":"值","f":"来源"}]}
|
||||
|
||||
类型（t）：
|
||||
- emo: 情绪状态（需要s主体）
|
||||
- loc: 地点/场景
|
||||
- act: 关键动作（需要s主体）
|
||||
- rev: 揭示/发现
|
||||
- ten: 冲突/张力
|
||||
- dec: 决定/承诺
|
||||
|
||||
规则：
|
||||
- s: 主体（谁）
|
||||
- v: 简洁值，10字内
|
||||
- f: "u"=用户发言中, "a"=角色回复中
|
||||
- 只提取对未来检索有价值的锚点
|
||||
- 无明显锚点返回空数组`;
|
||||
|
||||
function buildSemantic(atom, userName, aiName) {
|
||||
const speaker = atom.f === 'u' ? userName : aiName;
|
||||
const s = atom.s || speaker;
|
||||
|
||||
switch (atom.t) {
|
||||
case 'emo': return `${s}感到${atom.v}`;
|
||||
case 'loc': return `场景：${atom.v}`;
|
||||
case 'act': return `${s}${atom.v}`;
|
||||
case 'rev': return `揭示：${atom.v}`;
|
||||
case 'ten': return `冲突：${atom.v}`;
|
||||
case 'dec': return `${s}决定${atom.v}`;
|
||||
default: return `${s} ${atom.v}`;
|
||||
}
|
||||
}
|
||||
|
||||
const sleep = (ms) => new Promise(r => setTimeout(r, ms));
|
||||
|
||||
async function extractAtomsForRoundWithRetry(userMessage, aiMessage, aiFloor, options = {}) {
|
||||
const { timeout = DEFAULT_TIMEOUT } = options;
|
||||
|
||||
if (!aiMessage?.mes?.trim()) return [];
|
||||
|
||||
const parts = [];
|
||||
const userName = userMessage?.name || '用户';
|
||||
const aiName = aiMessage.name || '角色';
|
||||
|
||||
if (userMessage?.mes?.trim()) {
|
||||
const userText = filterText(userMessage.mes);
|
||||
parts.push(`【用户：${userName}】\n${userText}`);
|
||||
}
|
||||
|
||||
const aiText = filterText(aiMessage.mes);
|
||||
parts.push(`【角色：${aiName}】\n${aiText}`);
|
||||
|
||||
const input = parts.join('\n\n---\n\n');
|
||||
|
||||
xbLog.info(MODULE_ID, `floor ${aiFloor} 发送输入 len=${input.length}`);
|
||||
|
||||
for (let attempt = 0; attempt <= RETRY_COUNT; attempt++) {
|
||||
if (batchCancelled) return [];
|
||||
|
||||
try {
|
||||
const response = await callLLM([
|
||||
{ role: 'system', content: SYSTEM_PROMPT },
|
||||
{ role: 'user', content: input },
|
||||
], {
|
||||
temperature: 0.2,
|
||||
max_tokens: 500,
|
||||
timeout,
|
||||
});
|
||||
|
||||
if (!response || !String(response).trim()) {
|
||||
xbLog.warn(MODULE_ID, `floor ${aiFloor} 解析失败：响应为空`);
|
||||
if (attempt < RETRY_COUNT) {
|
||||
await sleep(RETRY_DELAY);
|
||||
continue;
|
||||
}
|
||||
return [];
|
||||
}
|
||||
|
||||
let parsed;
|
||||
try {
|
||||
parsed = parseJson(response);
|
||||
} catch (e) {
|
||||
xbLog.warn(MODULE_ID, `floor ${aiFloor} 解析失败：JSON 异常`);
|
||||
if (attempt < RETRY_COUNT) {
|
||||
await sleep(RETRY_DELAY);
|
||||
continue;
|
||||
}
|
||||
return [];
|
||||
}
|
||||
|
||||
if (!parsed?.atoms || !Array.isArray(parsed.atoms)) {
|
||||
xbLog.warn(MODULE_ID, `floor ${aiFloor} 解析失败：atoms 缺失`);
|
||||
if (attempt < RETRY_COUNT) {
|
||||
await sleep(RETRY_DELAY);
|
||||
continue;
|
||||
}
|
||||
return [];
|
||||
}
|
||||
|
||||
return parsed.atoms
|
||||
.filter(a => a?.t && a?.v)
|
||||
.map((a, idx) => ({
|
||||
atomId: `atom-${aiFloor}-${idx}`,
|
||||
floor: aiFloor,
|
||||
type: a.t,
|
||||
subject: a.s || null,
|
||||
value: String(a.v).slice(0, 30),
|
||||
source: a.f === 'u' ? 'user' : 'ai',
|
||||
semantic: buildSemantic(a, userName, aiName),
|
||||
}));
|
||||
|
||||
} catch (e) {
|
||||
if (batchCancelled) return [];
|
||||
|
||||
if (attempt < RETRY_COUNT) {
|
||||
xbLog.warn(MODULE_ID, `floor ${aiFloor} 第${attempt + 1}次失败，重试...`, e?.message);
|
||||
await sleep(RETRY_DELAY * (attempt + 1));
|
||||
continue;
|
||||
}
|
||||
xbLog.error(MODULE_ID, `floor ${aiFloor} 失败`, e);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
return [];
|
||||
}
|
||||
|
||||
/**
|
||||
* 单轮配对提取（增量时使用）
|
||||
*/
|
||||
export async function extractAtomsForRound(userMessage, aiMessage, aiFloor, options = {}) {
|
||||
return extractAtomsForRoundWithRetry(userMessage, aiMessage, aiFloor, options);
|
||||
}
|
||||
|
||||
/**
|
||||
* 批量提取（首批 staggered 启动）
|
||||
* @param {Array} chat
|
||||
* @param {Function} onProgress - (current, total, failed) => void
|
||||
*/
|
||||
export async function batchExtractAtoms(chat, onProgress) {
|
||||
if (!chat?.length) return [];
|
||||
|
||||
batchCancelled = false;
|
||||
|
||||
const pairs = [];
|
||||
for (let i = 0; i < chat.length; i++) {
|
||||
if (!chat[i].is_user) {
|
||||
const userMsg = (i > 0 && chat[i - 1]?.is_user) ? chat[i - 1] : null;
|
||||
pairs.push({ userMsg, aiMsg: chat[i], aiFloor: i });
|
||||
}
|
||||
}
|
||||
|
||||
if (!pairs.length) return [];
|
||||
|
||||
const allAtoms = [];
|
||||
let completed = 0;
|
||||
let failed = 0;
|
||||
|
||||
for (let i = 0; i < pairs.length; i += CONCURRENCY) {
|
||||
if (batchCancelled) {
|
||||
xbLog.info(MODULE_ID, `批量提取已取消 (${completed}/${pairs.length})`);
|
||||
break;
|
||||
}
|
||||
|
||||
const batch = pairs.slice(i, i + CONCURRENCY);
|
||||
|
||||
// ★ 首批 staggered 启动：错开 80ms 发送
|
||||
if (i === 0) {
|
||||
const promises = batch.map((pair, idx) => (async () => {
|
||||
await sleep(idx * STAGGER_DELAY);
|
||||
|
||||
if (batchCancelled) return;
|
||||
|
||||
try {
|
||||
const atoms = await extractAtomsForRoundWithRetry(pair.userMsg, pair.aiMsg, pair.aiFloor, { timeout: DEFAULT_TIMEOUT });
|
||||
if (atoms?.length) {
|
||||
allAtoms.push(...atoms);
|
||||
} else {
|
||||
failed++;
|
||||
}
|
||||
} catch {
|
||||
failed++;
|
||||
}
|
||||
completed++;
|
||||
onProgress?.(completed, pairs.length, failed);
|
||||
})());
|
||||
await Promise.all(promises);
|
||||
} else {
|
||||
// 后续批次正常并行
|
||||
const promises = batch.map(pair =>
|
||||
extractAtomsForRoundWithRetry(pair.userMsg, pair.aiMsg, pair.aiFloor, { timeout: DEFAULT_TIMEOUT })
|
||||
.then(atoms => {
|
||||
if (batchCancelled) return;
|
||||
if (atoms?.length) {
|
||||
allAtoms.push(...atoms);
|
||||
} else {
|
||||
failed++;
|
||||
}
|
||||
completed++;
|
||||
onProgress?.(completed, pairs.length, failed);
|
||||
})
|
||||
.catch(() => {
|
||||
if (batchCancelled) return;
|
||||
failed++;
|
||||
completed++;
|
||||
onProgress?.(completed, pairs.length, failed);
|
||||
})
|
||||
);
|
||||
|
||||
await Promise.all(promises);
|
||||
}
|
||||
|
||||
// 批次间隔
|
||||
if (i + CONCURRENCY < pairs.length && !batchCancelled) {
|
||||
await sleep(30);
|
||||
}
|
||||
}
|
||||
|
||||
const status = batchCancelled ? '已取消' : '完成';
|
||||
xbLog.info(MODULE_ID, `批量提取${status}: ${allAtoms.length} atoms, ${completed}/${pairs.length}, ${failed} 失败`);
|
||||
|
||||
return allAtoms;
|
||||
}
|
||||
72
modules/story-summary/vector/llm/llm-service.js
Normal file
72
modules/story-summary/vector/llm/llm-service.js
Normal file
@@ -0,0 +1,72 @@
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
// vector/llm/llm-service.js
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
|
||||
import { xbLog } from '../../../../core/debug-core.js';
|
||||
|
||||
const MODULE_ID = 'vector-llm-service';
|
||||
|
||||
// 唯一 ID 计数器
|
||||
let callCounter = 0;
|
||||
|
||||
function getStreamingModule() {
|
||||
const mod = window.xiaobaixStreamingGeneration;
|
||||
return mod?.xbgenrawCommand ? mod : null;
|
||||
}
|
||||
|
||||
function generateUniqueId(prefix = 'llm') {
|
||||
callCounter = (callCounter + 1) % 100000;
|
||||
return `${prefix}-${callCounter}-${Date.now().toString(36)}`;
|
||||
}
|
||||
|
||||
function b64UrlEncode(str) {
|
||||
const utf8 = new TextEncoder().encode(String(str));
|
||||
let bin = '';
|
||||
utf8.forEach(b => bin += String.fromCharCode(b));
|
||||
return btoa(bin).replace(/\+/g, '-').replace(/\//g, '_').replace(/=+$/, '');
|
||||
}
|
||||
|
||||
/**
|
||||
* 统一LLM调用 - 走酒馆后端（非流式）
|
||||
*/
|
||||
export async function callLLM(messages, options = {}) {
|
||||
const {
|
||||
temperature = 0.2,
|
||||
max_tokens = 500,
|
||||
} = options;
|
||||
|
||||
const mod = getStreamingModule();
|
||||
if (!mod) throw new Error('生成模块未加载');
|
||||
|
||||
const top64 = b64UrlEncode(JSON.stringify(messages));
|
||||
|
||||
// ★ 每次调用用唯一 ID，避免 session 冲突
|
||||
const uniqueId = generateUniqueId('l0');
|
||||
|
||||
const args = {
|
||||
as: 'user',
|
||||
nonstream: 'true',
|
||||
top64,
|
||||
id: uniqueId,
|
||||
temperature: String(temperature),
|
||||
max_tokens: String(max_tokens),
|
||||
};
|
||||
|
||||
try {
|
||||
// 非流式直接返回结果
|
||||
const result = await mod.xbgenrawCommand(args, '');
|
||||
return String(result ?? '');
|
||||
} catch (e) {
|
||||
xbLog.error(MODULE_ID, 'LLM调用失败', e);
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
export function parseJson(text) {
|
||||
if (!text) return null;
|
||||
let s = text.trim().replace(/^```(?:json)?\s*/i, '').replace(/\s*```$/i, '').trim();
|
||||
try { return JSON.parse(s); } catch { }
|
||||
const i = s.indexOf('{'), j = s.lastIndexOf('}');
|
||||
if (i !== -1 && j > i) try { return JSON.parse(s.slice(i, j + 1)); } catch { }
|
||||
return null;
|
||||
}
|
||||
102
modules/story-summary/vector/llm/query-expansion.js
Normal file
102
modules/story-summary/vector/llm/query-expansion.js
Normal file
@@ -0,0 +1,102 @@
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
// query-expansion.js - 完整输入，不截断
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
|
||||
import { callLLM, parseJson } from './llm-service.js';
|
||||
import { xbLog } from '../../../../core/debug-core.js';
|
||||
import { filterText } from '../utils/text-filter.js';
|
||||
|
||||
const MODULE_ID = 'query-expansion';
|
||||
const SESSION_ID = 'xb6';
|
||||
|
||||
const SYSTEM_PROMPT = `你是检索词生成器。根据最近对话，输出用于检索历史剧情的关键词。
|
||||
|
||||
只输出JSON：
|
||||
{"e":["显式人物/地名"],"i":["隐含人物/情绪/话题"],"q":["检索短句"]}
|
||||
|
||||
规则：
|
||||
- e: 对话中明确提到的人名/地名，1-4个
|
||||
- i: 推断出的相关人物/情绪/话题，1-5个
|
||||
- q: 用于向量检索的短句，2-3个，每个15字内
|
||||
- 关注：正在讨论什么、涉及谁、情绪氛围`;
|
||||
|
||||
/**
|
||||
* Query Expansion
|
||||
* @param {Array} messages - 完整消息数组（最后2-3轮）
|
||||
*/
|
||||
export async function expandQuery(messages, options = {}) {
|
||||
const { timeout = 6000 } = options;
|
||||
|
||||
if (!messages?.length) {
|
||||
return { entities: [], implicit: [], queries: [] };
|
||||
}
|
||||
|
||||
// 完整格式化，不截断
|
||||
const input = messages.map(m => {
|
||||
const speaker = m.is_user ? '用户' : (m.name || '角色');
|
||||
const text = filterText(m.mes || '').trim();
|
||||
return `【${speaker}】\n${text}`;
|
||||
}).join('\n\n');
|
||||
|
||||
const T0 = performance.now();
|
||||
|
||||
try {
|
||||
const response = await callLLM([
|
||||
{ role: 'system', content: SYSTEM_PROMPT },
|
||||
{ role: 'user', content: input },
|
||||
], {
|
||||
temperature: 0.15,
|
||||
max_tokens: 250,
|
||||
timeout,
|
||||
sessionId: SESSION_ID,
|
||||
});
|
||||
|
||||
const parsed = parseJson(response);
|
||||
if (!parsed) {
|
||||
xbLog.warn(MODULE_ID, 'JSON解析失败', response?.slice(0, 200));
|
||||
return { entities: [], implicit: [], queries: [] };
|
||||
}
|
||||
|
||||
const result = {
|
||||
entities: Array.isArray(parsed.e) ? parsed.e.slice(0, 5) : [],
|
||||
implicit: Array.isArray(parsed.i) ? parsed.i.slice(0, 6) : [],
|
||||
queries: Array.isArray(parsed.q) ? parsed.q.slice(0, 4) : [],
|
||||
};
|
||||
|
||||
xbLog.info(MODULE_ID, `完成 (${Math.round(performance.now() - T0)}ms) e=${result.entities.length} i=${result.implicit.length} q=${result.queries.length}`);
|
||||
return result;
|
||||
|
||||
} catch (e) {
|
||||
xbLog.error(MODULE_ID, '调用失败', e);
|
||||
return { entities: [], implicit: [], queries: [] };
|
||||
}
|
||||
}
|
||||
|
||||
// 缓存
|
||||
const cache = new Map();
|
||||
const CACHE_TTL = 300000;
|
||||
|
||||
function hashMessages(messages) {
|
||||
const text = messages.slice(-2).map(m => (m.mes || '').slice(0, 100)).join('|');
|
||||
let h = 0;
|
||||
for (let i = 0; i < text.length; i++) h = ((h << 5) - h + text.charCodeAt(i)) | 0;
|
||||
return h.toString(36);
|
||||
}
|
||||
|
||||
export async function expandQueryCached(messages, options = {}) {
|
||||
const key = hashMessages(messages);
|
||||
const cached = cache.get(key);
|
||||
if (cached && Date.now() - cached.time < CACHE_TTL) return cached.result;
|
||||
|
||||
const result = await expandQuery(messages, options);
|
||||
if (result.entities.length || result.queries.length) {
|
||||
if (cache.size > 50) cache.delete(cache.keys().next().value);
|
||||
cache.set(key, { result, time: Date.now() });
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
export function buildSearchText(expansion) {
|
||||
return [...(expansion.entities || []), ...(expansion.implicit || []), ...(expansion.queries || [])]
|
||||
.filter(Boolean).join(' ');
|
||||
}
|
||||
59
modules/story-summary/vector/llm/siliconflow.js
Normal file
59
modules/story-summary/vector/llm/siliconflow.js
Normal file
@@ -0,0 +1,59 @@
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
// siliconflow.js - 仅保留 Embedding
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
|
||||
const BASE_URL = 'https://api.siliconflow.cn';
|
||||
const EMBEDDING_MODEL = 'BAAI/bge-m3';
|
||||
|
||||
export function getApiKey() {
|
||||
try {
|
||||
const raw = localStorage.getItem('summary_panel_config');
|
||||
if (raw) {
|
||||
const parsed = JSON.parse(raw);
|
||||
return parsed.vector?.online?.key || null;
|
||||
}
|
||||
} catch { }
|
||||
return null;
|
||||
}
|
||||
|
||||
export async function embed(texts, options = {}) {
|
||||
if (!texts?.length) return [];
|
||||
|
||||
const key = getApiKey();
|
||||
if (!key) throw new Error('未配置硅基 API Key');
|
||||
|
||||
const { timeout = 30000, signal } = options;
|
||||
const controller = new AbortController();
|
||||
const timeoutId = setTimeout(() => controller.abort(), timeout);
|
||||
|
||||
try {
|
||||
const response = await fetch(`${BASE_URL}/v1/embeddings`, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Authorization': `Bearer ${key}`,
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify({
|
||||
model: EMBEDDING_MODEL,
|
||||
input: texts,
|
||||
}),
|
||||
signal: signal || controller.signal,
|
||||
});
|
||||
|
||||
clearTimeout(timeoutId);
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text().catch(() => '');
|
||||
throw new Error(`Embedding ${response.status}: ${errorText.slice(0, 200)}`);
|
||||
}
|
||||
|
||||
const data = await response.json();
|
||||
return (data.data || [])
|
||||
.sort((a, b) => a.index - b.index)
|
||||
.map(item => Array.isArray(item.embedding) ? item.embedding : Array.from(item.embedding));
|
||||
} finally {
|
||||
clearTimeout(timeoutId);
|
||||
}
|
||||
}
|
||||
|
||||
export { EMBEDDING_MODEL as MODELS };
|
||||
@@ -1,4 +1,4 @@
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
// Story Summary - Chunk Builder
|
||||
// 标准 RAG chunking: ~200 tokens per chunk
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
@@ -19,6 +19,7 @@ import {
|
||||
import { embed, getEngineFingerprint } from '../utils/embedder.js';
|
||||
import { xbLog } from '../../../../core/debug-core.js';
|
||||
import { filterText } from '../utils/text-filter.js';
|
||||
import { extractAndStoreAtomsForRound } from './state-integration.js';
|
||||
|
||||
const MODULE_ID = 'chunk-builder';
|
||||
|
||||
@@ -201,8 +202,7 @@ export async function buildAllChunks(options = {}) {
|
||||
await saveChunks(chatId, allChunks);
|
||||
|
||||
const texts = allChunks.map(c => c.text);
|
||||
const isLocal = vectorConfig.engine === 'local';
|
||||
const batchSize = isLocal ? 5 : 20;
|
||||
const batchSize = 20;
|
||||
|
||||
let completed = 0;
|
||||
let errors = 0;
|
||||
@@ -302,6 +302,7 @@ export async function buildIncrementalChunks(options = {}) {
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
// L1 同步（消息变化时调用）
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
@@ -337,13 +338,6 @@ export async function syncOnMessageReceived(chatId, lastFloor, message, vectorCo
|
||||
if (!chatId || lastFloor < 0 || !message) return;
|
||||
if (!vectorConfig?.enabled) return;
|
||||
|
||||
// 本地模型未加载时跳过（避免意外触发下载或报错）
|
||||
if (vectorConfig.engine === "local") {
|
||||
const { isLocalModelLoaded, DEFAULT_LOCAL_MODEL } = await import("../utils/embedder.js");
|
||||
const modelId = vectorConfig.local?.modelId || DEFAULT_LOCAL_MODEL;
|
||||
if (!isLocalModelLoaded(modelId)) return;
|
||||
}
|
||||
|
||||
// 删除该楼层旧的
|
||||
await deleteChunksAtFloor(chatId, lastFloor);
|
||||
|
||||
@@ -367,4 +361,18 @@ export async function syncOnMessageReceived(chatId, lastFloor, message, vectorCo
|
||||
} catch (e) {
|
||||
xbLog.error(MODULE_ID, `消息同步失败：floor ${lastFloor}`, e);
|
||||
}
|
||||
// L0 配对提取（仅 AI 消息触发）
|
||||
if (!message.is_user) {
|
||||
const { chat } = getContext();
|
||||
const userFloor = lastFloor - 1;
|
||||
const userMessage = (userFloor >= 0 && chat[userFloor]?.is_user) ? chat[userFloor] : null;
|
||||
|
||||
try {
|
||||
await extractAndStoreAtomsForRound(lastFloor, message, userMessage);
|
||||
} catch (e) {
|
||||
xbLog.warn(MODULE_ID, `Atom 提取失败: floor ${lastFloor}`, e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
// Story Summary - State Integration (L0)
|
||||
// 事件监听 + 回滚钩子注册
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
// ============================================================================
|
||||
// state-integration.js - L0 记忆锚点管理
|
||||
// 支持增量提取、清空、取消
|
||||
// ============================================================================
|
||||
|
||||
import { getContext } from '../../../../../../../extensions.js';
|
||||
import { xbLog } from '../../../../core/debug-core.js';
|
||||
@@ -11,70 +11,174 @@ import {
|
||||
deleteStateAtomsFromFloor,
|
||||
deleteStateVectorsFromFloor,
|
||||
getStateAtoms,
|
||||
clearStateAtoms,
|
||||
clearStateVectors,
|
||||
getL0FloorStatus,
|
||||
setL0FloorStatus,
|
||||
clearL0Index,
|
||||
deleteL0IndexFromFloor,
|
||||
} from '../storage/state-store.js';
|
||||
import { embed, getEngineFingerprint } from '../utils/embedder.js';
|
||||
import { embed } from '../llm/siliconflow.js';
|
||||
import { extractAtomsForRound, cancelBatchExtraction } from '../llm/atom-extraction.js';
|
||||
import { getVectorConfig } from '../../data/config.js';
|
||||
import { getEngineFingerprint } from '../utils/embedder.js';
|
||||
import { filterText } from '../utils/text-filter.js';
|
||||
|
||||
const MODULE_ID = 'state-integration';
|
||||
|
||||
let initialized = false;
|
||||
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
export function cancelL0Extraction() {
|
||||
cancelBatchExtraction();
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// 初始化
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
// ============================================================================
|
||||
|
||||
export function initStateIntegration() {
|
||||
if (initialized) return;
|
||||
initialized = true;
|
||||
|
||||
// 监听变量团队的事件
|
||||
$(document).on('xiaobaix:variables:stateAtomsGenerated', handleStateAtomsGenerated);
|
||||
|
||||
// 注册回滚钩子
|
||||
globalThis.LWB_StateRollbackHook = handleStateRollback;
|
||||
|
||||
xbLog.info(MODULE_ID, 'L0 状态层集成已初始化');
|
||||
}
|
||||
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
// 事件处理
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
// ============================================================================
|
||||
// 统计
|
||||
// ============================================================================
|
||||
|
||||
async function handleStateAtomsGenerated(e, data) {
|
||||
const { atoms } = data || {};
|
||||
if (!atoms?.length) return;
|
||||
|
||||
const { chatId } = getContext();
|
||||
if (!chatId) return;
|
||||
|
||||
const validAtoms = atoms.filter(a => a?.chatId === chatId);
|
||||
if (!validAtoms.length) {
|
||||
xbLog.warn(MODULE_ID, `atoms.chatId 不匹配，期望 ${chatId}，跳过`);
|
||||
return;
|
||||
export async function getAnchorStats() {
|
||||
const { chat } = getContext();
|
||||
if (!chat?.length) {
|
||||
return { extracted: 0, total: 0, pending: 0, empty: 0, fail: 0 };
|
||||
}
|
||||
|
||||
xbLog.info(MODULE_ID, `收到 ${validAtoms.length} 个 StateAtom`);
|
||||
|
||||
// 1. 存入 chat_metadata（持久化）
|
||||
saveStateAtoms(validAtoms);
|
||||
|
||||
// 2. 向量化并存入 IndexedDB
|
||||
const vectorCfg = getVectorConfig();
|
||||
if (!vectorCfg?.enabled) {
|
||||
xbLog.info(MODULE_ID, '向量未启用，跳过 L0 向量化');
|
||||
return;
|
||||
const aiFloors = [];
|
||||
for (let i = 0; i < chat.length; i++) {
|
||||
if (!chat[i]?.is_user) aiFloors.push(i);
|
||||
}
|
||||
|
||||
await vectorizeAtoms(chatId, validAtoms, vectorCfg);
|
||||
let ok = 0;
|
||||
let empty = 0;
|
||||
let fail = 0;
|
||||
|
||||
for (const f of aiFloors) {
|
||||
const s = getL0FloorStatus(f);
|
||||
if (!s) continue;
|
||||
if (s.status === 'ok') ok++;
|
||||
else if (s.status === 'empty') empty++;
|
||||
else if (s.status === 'fail') fail++;
|
||||
}
|
||||
|
||||
const total = aiFloors.length;
|
||||
const completed = ok + empty;
|
||||
const pending = Math.max(0, total - completed);
|
||||
|
||||
return { extracted: completed, total, pending, empty, fail };
|
||||
}
|
||||
|
||||
async function vectorizeAtoms(chatId, atoms, vectorCfg) {
|
||||
// ============================================================================
|
||||
// 增量提取
|
||||
// ============================================================================
|
||||
|
||||
function buildL0InputText(userMessage, aiMessage) {
|
||||
const parts = [];
|
||||
const userName = userMessage?.name || '用户';
|
||||
const aiName = aiMessage?.name || '角色';
|
||||
|
||||
if (userMessage?.mes?.trim()) {
|
||||
parts.push(`【用户：${userName}】\n${filterText(userMessage.mes).trim()}`);
|
||||
}
|
||||
if (aiMessage?.mes?.trim()) {
|
||||
parts.push(`【角色：${aiName}】\n${filterText(aiMessage.mes).trim()}`);
|
||||
}
|
||||
|
||||
return parts.join('\n\n---\n\n').trim();
|
||||
}
|
||||
|
||||
export async function incrementalExtractAtoms(chatId, chat, onProgress) {
|
||||
if (!chatId || !chat?.length) return { built: 0 };
|
||||
|
||||
const vectorCfg = getVectorConfig();
|
||||
if (!vectorCfg?.enabled) return { built: 0 };
|
||||
|
||||
const pendingPairs = [];
|
||||
|
||||
for (let i = 0; i < chat.length; i++) {
|
||||
const msg = chat[i];
|
||||
if (!msg || msg.is_user) continue;
|
||||
|
||||
const st = getL0FloorStatus(i);
|
||||
if (st?.status === 'ok' || st?.status === 'empty') {
|
||||
continue;
|
||||
}
|
||||
|
||||
const userMsg = (i > 0 && chat[i - 1]?.is_user) ? chat[i - 1] : null;
|
||||
const inputText = buildL0InputText(userMsg, msg);
|
||||
|
||||
if (!inputText) {
|
||||
setL0FloorStatus(i, { status: 'empty', reason: 'filtered_empty', atoms: 0 });
|
||||
continue;
|
||||
}
|
||||
|
||||
pendingPairs.push({ userMsg, aiMsg: msg, aiFloor: i });
|
||||
}
|
||||
|
||||
if (!pendingPairs.length) {
|
||||
onProgress?.(0, 0, '已全部提取');
|
||||
return { built: 0 };
|
||||
}
|
||||
|
||||
xbLog.info(MODULE_ID, `增量 L0 提取：pending=${pendingPairs.length}`);
|
||||
|
||||
let completed = 0;
|
||||
const total = pendingPairs.length;
|
||||
let builtAtoms = 0;
|
||||
|
||||
for (const pair of pendingPairs) {
|
||||
const floor = pair.aiFloor;
|
||||
const prev = getL0FloorStatus(floor);
|
||||
|
||||
try {
|
||||
const atoms = await extractAtomsForRound(pair.userMsg, pair.aiMsg, floor, { timeout: 20000 });
|
||||
|
||||
if (!atoms?.length) {
|
||||
setL0FloorStatus(floor, { status: 'empty', reason: 'llm_empty', atoms: 0 });
|
||||
} else {
|
||||
atoms.forEach(a => a.chatId = chatId);
|
||||
saveStateAtoms(atoms);
|
||||
await vectorizeAtoms(chatId, atoms);
|
||||
|
||||
setL0FloorStatus(floor, { status: 'ok', atoms: atoms.length });
|
||||
builtAtoms += atoms.length;
|
||||
}
|
||||
} catch (e) {
|
||||
setL0FloorStatus(floor, {
|
||||
status: 'fail',
|
||||
attempts: (prev?.attempts || 0) + 1,
|
||||
reason: String(e?.message || e).replace(/\s+/g, ' ').slice(0, 120),
|
||||
});
|
||||
} finally {
|
||||
completed++;
|
||||
onProgress?.(`L0: ${completed}/${total}`, completed, total);
|
||||
}
|
||||
}
|
||||
|
||||
xbLog.info(MODULE_ID, `增量 L0 完成：atoms=${builtAtoms}, floors=${pendingPairs.length}`);
|
||||
return { built: builtAtoms };
|
||||
}
|
||||
|
||||
async function vectorizeAtoms(chatId, atoms) {
|
||||
if (!atoms?.length) return;
|
||||
|
||||
const vectorCfg = getVectorConfig();
|
||||
if (!vectorCfg?.enabled) return;
|
||||
|
||||
const texts = atoms.map(a => a.semantic);
|
||||
const fingerprint = getEngineFingerprint(vectorCfg);
|
||||
|
||||
try {
|
||||
const vectors = await embed(texts, vectorCfg);
|
||||
const vectors = await embed(texts, { timeout: 30000 });
|
||||
|
||||
const items = atoms.map((a, i) => ({
|
||||
atomId: a.atomId,
|
||||
@@ -83,34 +187,106 @@ async function vectorizeAtoms(chatId, atoms, vectorCfg) {
|
||||
}));
|
||||
|
||||
await saveStateVectors(chatId, items, fingerprint);
|
||||
xbLog.info(MODULE_ID, `L0 向量化完成: ${items.length} 个`);
|
||||
xbLog.info(MODULE_ID, `L0 向量化完成: ${items.length} 条`);
|
||||
} catch (e) {
|
||||
xbLog.error(MODULE_ID, 'L0 向量化失败', e);
|
||||
// 不阻塞，向量可后续通过"生成向量"重建
|
||||
}
|
||||
}
|
||||
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
// ============================================================================
|
||||
// 清空
|
||||
// ============================================================================
|
||||
|
||||
export async function clearAllAtomsAndVectors(chatId) {
|
||||
clearStateAtoms();
|
||||
clearL0Index();
|
||||
if (chatId) {
|
||||
await clearStateVectors(chatId);
|
||||
}
|
||||
xbLog.info(MODULE_ID, '已清空所有记忆锚点');
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// 实时增量（AI 消息后触发）- 保留原有逻辑
|
||||
// ============================================================================
|
||||
|
||||
let extractionQueue = [];
|
||||
let isProcessing = false;
|
||||
|
||||
export async function extractAndStoreAtomsForRound(aiFloor, aiMessage, userMessage) {
|
||||
const { chatId } = getContext();
|
||||
if (!chatId) return;
|
||||
|
||||
const vectorCfg = getVectorConfig();
|
||||
if (!vectorCfg?.enabled) return;
|
||||
|
||||
extractionQueue.push({ aiFloor, aiMessage, userMessage, chatId });
|
||||
processQueue();
|
||||
}
|
||||
|
||||
async function processQueue() {
|
||||
if (isProcessing || extractionQueue.length === 0) return;
|
||||
isProcessing = true;
|
||||
|
||||
while (extractionQueue.length > 0) {
|
||||
const { aiFloor, aiMessage, userMessage, chatId } = extractionQueue.shift();
|
||||
|
||||
try {
|
||||
const atoms = await extractAtomsForRound(userMessage, aiMessage, aiFloor, { timeout: 12000 });
|
||||
|
||||
if (!atoms?.length) {
|
||||
xbLog.info(MODULE_ID, `floor ${aiFloor}: 无有效 atoms`);
|
||||
continue;
|
||||
}
|
||||
|
||||
atoms.forEach(a => a.chatId = chatId);
|
||||
saveStateAtoms(atoms);
|
||||
await vectorizeAtoms(chatId, atoms);
|
||||
|
||||
xbLog.info(MODULE_ID, `floor ${aiFloor}: ${atoms.length} atoms 已存储`);
|
||||
} catch (e) {
|
||||
xbLog.error(MODULE_ID, `floor ${aiFloor} 处理失败`, e);
|
||||
}
|
||||
}
|
||||
|
||||
isProcessing = false;
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// 回滚钩子
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
// ============================================================================
|
||||
|
||||
async function handleStateRollback(floor) {
|
||||
xbLog.info(MODULE_ID, `收到回滚请求: floor >= ${floor}`);
|
||||
|
||||
const { chatId } = getContext();
|
||||
|
||||
// 1. 删除 chat_metadata 中的 atoms
|
||||
deleteStateAtomsFromFloor(floor);
|
||||
deleteL0IndexFromFloor(floor);
|
||||
|
||||
// 2. 删除 IndexedDB 中的 vectors
|
||||
if (chatId) {
|
||||
await deleteStateVectorsFromFloor(chatId, floor);
|
||||
}
|
||||
}
|
||||
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
// 重建向量（供"生成向量"按钮调用）
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
// ============================================================================
|
||||
// 兼容旧接口
|
||||
// ============================================================================
|
||||
|
||||
export async function batchExtractAndStoreAtoms(chatId, chat, onProgress) {
|
||||
if (!chatId || !chat?.length) return { built: 0 };
|
||||
|
||||
const vectorCfg = getVectorConfig();
|
||||
if (!vectorCfg?.enabled) return { built: 0 };
|
||||
|
||||
xbLog.info(MODULE_ID, `开始批量 L0 提取: ${chat.length} 条消息`);
|
||||
|
||||
clearStateAtoms();
|
||||
clearL0Index();
|
||||
await clearStateVectors(chatId);
|
||||
|
||||
return await incrementalExtractAtoms(chatId, chat, onProgress);
|
||||
}
|
||||
|
||||
export async function rebuildStateVectors(chatId, vectorCfg) {
|
||||
if (!chatId || !vectorCfg?.enabled) return { built: 0 };
|
||||
@@ -118,36 +294,10 @@ export async function rebuildStateVectors(chatId, vectorCfg) {
|
||||
const atoms = getStateAtoms();
|
||||
if (!atoms.length) return { built: 0 };
|
||||
|
||||
xbLog.info(MODULE_ID, `开始重建 L0 向量: ${atoms.length} 个 atom`);
|
||||
xbLog.info(MODULE_ID, `重建 L0 向量: ${atoms.length} 条 atom`);
|
||||
|
||||
// 清空旧向量
|
||||
await clearStateVectors(chatId);
|
||||
await vectorizeAtoms(chatId, atoms);
|
||||
|
||||
// 重新向量化
|
||||
const fingerprint = getEngineFingerprint(vectorCfg);
|
||||
const batchSize = vectorCfg.engine === 'local' ? 5 : 25;
|
||||
let built = 0;
|
||||
|
||||
for (let i = 0; i < atoms.length; i += batchSize) {
|
||||
const batch = atoms.slice(i, i + batchSize);
|
||||
const texts = batch.map(a => a.semantic);
|
||||
|
||||
try {
|
||||
const vectors = await embed(texts, vectorCfg);
|
||||
|
||||
const items = batch.map((a, j) => ({
|
||||
atomId: a.atomId,
|
||||
floor: a.floor,
|
||||
vector: vectors[j],
|
||||
}));
|
||||
|
||||
await saveStateVectors(chatId, items, fingerprint);
|
||||
built += items.length;
|
||||
} catch (e) {
|
||||
xbLog.error(MODULE_ID, `L0 向量化批次失败: ${i}-${i + batchSize}`, e);
|
||||
}
|
||||
}
|
||||
|
||||
xbLog.info(MODULE_ID, `L0 向量重建完成: ${built}/${atoms.length}`);
|
||||
return { built };
|
||||
return { built: atoms.length };
|
||||
}
|
||||
|
||||
@@ -1,129 +0,0 @@
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
// Entity Recognition & Relation Graph
|
||||
// 实体识别与关系扩散
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
|
||||
/**
|
||||
* 从文本中匹配已知实体
|
||||
* @param {string} text - 待匹配文本
|
||||
* @param {Set<string>} knownEntities - 已知实体集合
|
||||
* @returns {string[]} - 匹配到的实体
|
||||
*/
|
||||
export function matchEntities(text, knownEntities) {
|
||||
if (!text || !knownEntities?.size) return [];
|
||||
|
||||
const matched = new Set();
|
||||
|
||||
for (const entity of knownEntities) {
|
||||
// 精确包含
|
||||
if (text.includes(entity)) {
|
||||
matched.add(entity);
|
||||
continue;
|
||||
}
|
||||
|
||||
// 处理简称：如果实体是"林黛玉"，文本包含"黛玉"
|
||||
if (entity.length >= 3) {
|
||||
const shortName = entity.slice(-2); // 取后两字
|
||||
if (text.includes(shortName)) {
|
||||
matched.add(entity);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return Array.from(matched);
|
||||
}
|
||||
|
||||
/**
|
||||
* 从角色数据和事件中收集所有已知实体
|
||||
*/
|
||||
export function collectKnownEntities(characters, events) {
|
||||
const entities = new Set();
|
||||
|
||||
// 从主要角色
|
||||
(characters?.main || []).forEach(m => {
|
||||
const name = typeof m === 'string' ? m : m.name;
|
||||
if (name) entities.add(name);
|
||||
});
|
||||
|
||||
// 从关系
|
||||
(characters?.relationships || []).forEach(r => {
|
||||
if (r.from) entities.add(r.from);
|
||||
if (r.to) entities.add(r.to);
|
||||
});
|
||||
|
||||
// 从事件参与者
|
||||
(events || []).forEach(e => {
|
||||
(e.participants || []).forEach(p => {
|
||||
if (p) entities.add(p);
|
||||
});
|
||||
});
|
||||
|
||||
return entities;
|
||||
}
|
||||
|
||||
/**
|
||||
* 构建关系邻接表
|
||||
* @param {Array} relationships - 关系数组
|
||||
* @returns {Map<string, Array<{target: string, weight: number}>>}
|
||||
*/
|
||||
export function buildRelationGraph(relationships) {
|
||||
const graph = new Map();
|
||||
|
||||
const trendWeight = {
|
||||
'交融': 1.0,
|
||||
'亲密': 0.9,
|
||||
'投缘': 0.7,
|
||||
'陌生': 0.3,
|
||||
'反感': 0.5,
|
||||
'厌恶': 0.6,
|
||||
'破裂': 0.7,
|
||||
};
|
||||
|
||||
for (const rel of relationships || []) {
|
||||
if (!rel.from || !rel.to) continue;
|
||||
|
||||
const weight = trendWeight[rel.trend] || 0.5;
|
||||
|
||||
// 双向
|
||||
if (!graph.has(rel.from)) graph.set(rel.from, []);
|
||||
if (!graph.has(rel.to)) graph.set(rel.to, []);
|
||||
|
||||
graph.get(rel.from).push({ target: rel.to, weight });
|
||||
graph.get(rel.to).push({ target: rel.from, weight });
|
||||
}
|
||||
|
||||
return graph;
|
||||
}
|
||||
|
||||
/**
|
||||
* 关系扩散（1跳）
|
||||
* @param {string[]} focusEntities - 焦点实体
|
||||
* @param {Map} graph - 关系图
|
||||
* @param {number} decayFactor - 衰减因子
|
||||
* @returns {Map<string, number>} - 实体 -> 激活分数
|
||||
*/
|
||||
export function spreadActivation(focusEntities, graph, decayFactor = 0.5) {
|
||||
const activation = new Map();
|
||||
|
||||
// 焦点实体初始分数 1.0
|
||||
for (const entity of focusEntities) {
|
||||
activation.set(entity, 1.0);
|
||||
}
|
||||
|
||||
// 1跳扩散
|
||||
for (const entity of focusEntities) {
|
||||
const neighbors = graph.get(entity) || [];
|
||||
|
||||
for (const { target, weight } of neighbors) {
|
||||
const spreadScore = weight * decayFactor;
|
||||
const existing = activation.get(target) || 0;
|
||||
|
||||
// 取最大值，不累加
|
||||
if (spreadScore > existing) {
|
||||
activation.set(target, spreadScore);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return activation;
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,237 +0,0 @@
|
||||
// text-search.js - 最终版
|
||||
|
||||
import MiniSearch from '../../../../libs/minisearch.mjs';
|
||||
|
||||
const STOP_WORDS = new Set([
|
||||
'的', '了', '是', '在', '和', '与', '或', '但', '而', '却',
|
||||
'这', '那', '他', '她', '它', '我', '你', '们', '着', '过',
|
||||
'把', '被', '给', '让', '向', '就', '都', '也', '还', '又',
|
||||
'很', '太', '更', '最', '只', '才', '已', '正', '会', '能',
|
||||
'要', '可', '得', '地', '之', '所', '以', '为', '于', '有',
|
||||
'不', '去', '来', '上', '下', '里', '说', '看', '吧', '呢',
|
||||
'啊', '吗', '呀', '哦', '嗯', '么',
|
||||
'の', 'に', 'は', 'を', 'が', 'と', 'で', 'へ', 'や', 'か',
|
||||
'も', 'な', 'よ', 'ね', 'わ', 'です', 'ます', 'した', 'ない',
|
||||
'the', 'a', 'an', 'is', 'are', 'was', 'were', 'be', 'been',
|
||||
'have', 'has', 'had', 'do', 'does', 'did', 'will', 'would',
|
||||
'to', 'of', 'in', 'on', 'at', 'for', 'with', 'by', 'from',
|
||||
'and', 'or', 'but', 'if', 'that', 'this', 'it', 'its',
|
||||
'i', 'you', 'he', 'she', 'we', 'they', 'my', 'your', 'his',
|
||||
]);
|
||||
|
||||
function tokenize(text) {
|
||||
const s = String(text || '').toLowerCase().trim();
|
||||
if (!s) return [];
|
||||
|
||||
const tokens = new Set();
|
||||
|
||||
// CJK Bigram + Trigram
|
||||
const cjk = s.match(/[\u4e00-\u9fff\u3400-\u4dbf]+/g) || [];
|
||||
for (const seg of cjk) {
|
||||
const chars = [...seg].filter(c => !STOP_WORDS.has(c));
|
||||
for (let i = 0; i < chars.length - 1; i++) {
|
||||
tokens.add(chars[i] + chars[i + 1]);
|
||||
}
|
||||
for (let i = 0; i < chars.length - 2; i++) {
|
||||
tokens.add(chars[i] + chars[i + 1] + chars[i + 2]);
|
||||
}
|
||||
}
|
||||
|
||||
// 日语假名
|
||||
const kana = s.match(/[\u3040-\u309f\u30a0-\u30ff]{2,}/g) || [];
|
||||
for (const k of kana) {
|
||||
if (!STOP_WORDS.has(k)) tokens.add(k);
|
||||
}
|
||||
|
||||
// 英文
|
||||
const en = s.match(/[a-z]{2,}/g) || [];
|
||||
for (const w of en) {
|
||||
if (!STOP_WORDS.has(w)) tokens.add(w);
|
||||
}
|
||||
|
||||
return [...tokens];
|
||||
}
|
||||
|
||||
let idx = null;
|
||||
let lastRevision = null;
|
||||
|
||||
function stripFloorTag(s) {
|
||||
return String(s || '').replace(/\s*\(#\d+(?:-\d+)?\)\s*$/, '').trim();
|
||||
}
|
||||
|
||||
export function ensureEventTextIndex(events, revision) {
|
||||
if (!events?.length) {
|
||||
idx = null;
|
||||
lastRevision = null;
|
||||
return;
|
||||
}
|
||||
if (idx && revision === lastRevision) return;
|
||||
|
||||
try {
|
||||
idx = new MiniSearch({
|
||||
fields: ['title', 'summary', 'participants'],
|
||||
storeFields: ['id'],
|
||||
tokenize,
|
||||
searchOptions: { tokenize },
|
||||
});
|
||||
|
||||
idx.addAll(events.map(e => ({
|
||||
id: e.id,
|
||||
title: e.title || '',
|
||||
summary: stripFloorTag(e.summary),
|
||||
participants: (e.participants || []).join(' '),
|
||||
})));
|
||||
|
||||
lastRevision = revision;
|
||||
} catch (e) {
|
||||
console.error('[text-search] Index build failed:', e);
|
||||
idx = null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* BM25 检索，返回 top-K 候选给 RRF
|
||||
*
|
||||
* 设计原则：
|
||||
* - 不做分数过滤（BM25 分数跨查询不可比）
|
||||
* - 不做匹配数过滤（bigram 让一个词产生多个 token）
|
||||
* - 只做 top-K（BM25 排序本身有区分度）
|
||||
* - 质量过滤交给 RRF 后的 hasVector 过滤
|
||||
*/
|
||||
/**
|
||||
* 动态 top-K：累积分数占比法
|
||||
*
|
||||
* 原理：BM25 分数服从幂律分布，少数高分条目贡献大部分总分
|
||||
* 取累积分数达到阈值的最小 K
|
||||
*
|
||||
* 参考：帕累托法则（80/20 法则）在信息检索中的应用
|
||||
*/
|
||||
export function dynamicTopK(scores, coverage = 0.90, minK = 15, maxK = 80) {
|
||||
if (!scores.length) return 0;
|
||||
|
||||
const total = scores.reduce((a, b) => a + b, 0);
|
||||
if (total <= 0) return Math.min(minK, scores.length);
|
||||
|
||||
let cumulative = 0;
|
||||
for (let i = 0; i < scores.length; i++) {
|
||||
cumulative += scores[i];
|
||||
if (cumulative / total >= coverage) {
|
||||
return Math.max(minK, Math.min(maxK, i + 1));
|
||||
}
|
||||
}
|
||||
|
||||
return Math.min(maxK, scores.length);
|
||||
}
|
||||
|
||||
export function searchEventsByText(queryText, limit = 80) {
|
||||
if (!idx || !queryText?.trim()) return [];
|
||||
|
||||
try {
|
||||
const results = idx.search(queryText, {
|
||||
boost: { title: 4, participants: 2, summary: 1 },
|
||||
fuzzy: false,
|
||||
prefix: false,
|
||||
});
|
||||
|
||||
if (!results.length) return [];
|
||||
|
||||
const scores = results.map(r => r.score);
|
||||
const k = dynamicTopK(scores, 0.90, 15, limit);
|
||||
|
||||
const output = results.slice(0, k).map((r, i) => ({
|
||||
id: r.id,
|
||||
textRank: i + 1,
|
||||
score: r.score,
|
||||
}));
|
||||
|
||||
const total = scores.reduce((a, b) => a + b, 0);
|
||||
const kCumulative = scores.slice(0, k).reduce((a, b) => a + b, 0);
|
||||
|
||||
output._gapInfo = {
|
||||
total: results.length,
|
||||
returned: k,
|
||||
coverage: ((kCumulative / total) * 100).toFixed(1) + '%',
|
||||
scoreRange: {
|
||||
top: scores[0]?.toFixed(1),
|
||||
cutoff: scores[k - 1]?.toFixed(1),
|
||||
p50: scores[Math.floor(scores.length / 2)]?.toFixed(1),
|
||||
last: scores[scores.length - 1]?.toFixed(1),
|
||||
},
|
||||
};
|
||||
|
||||
return output;
|
||||
|
||||
} catch (e) {
|
||||
console.error('[text-search] Search failed:', e);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
export function clearEventTextIndex() {
|
||||
idx = null;
|
||||
lastRevision = null;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Chunk 文本索引（待整理区 L1 补充）
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
let chunkIdx = null;
|
||||
let chunkIdxRevision = null;
|
||||
|
||||
export function ensureChunkTextIndex(chunks, revision) {
|
||||
if (chunkIdx && revision === chunkIdxRevision) return;
|
||||
|
||||
try {
|
||||
chunkIdx = new MiniSearch({
|
||||
fields: ['text'],
|
||||
storeFields: ['chunkId', 'floor'],
|
||||
tokenize,
|
||||
searchOptions: { tokenize },
|
||||
});
|
||||
|
||||
chunkIdx.addAll(chunks.map(c => ({
|
||||
id: c.chunkId,
|
||||
chunkId: c.chunkId,
|
||||
floor: c.floor,
|
||||
text: c.text || '',
|
||||
})));
|
||||
|
||||
chunkIdxRevision = revision;
|
||||
} catch (e) {
|
||||
console.error('[text-search] Chunk index build failed:', e);
|
||||
chunkIdx = null;
|
||||
}
|
||||
}
|
||||
|
||||
export function searchChunksByText(query, floorMin, floorMax, limit = 20) {
|
||||
if (!chunkIdx || !query?.trim()) return [];
|
||||
|
||||
try {
|
||||
const results = chunkIdx.search(query, {
|
||||
fuzzy: false,
|
||||
prefix: false,
|
||||
});
|
||||
|
||||
const filtered = results.filter(r => r.floor >= floorMin && r.floor <= floorMax);
|
||||
if (!filtered.length) return [];
|
||||
|
||||
const scores = filtered.map(r => r.score);
|
||||
const k = dynamicTopK(scores, 0.85, 5, limit);
|
||||
|
||||
return filtered.slice(0, k).map((r, i) => ({
|
||||
chunkId: r.chunkId,
|
||||
floor: r.floor,
|
||||
textRank: i + 1,
|
||||
score: r.score,
|
||||
}));
|
||||
} catch (e) {
|
||||
console.error('[text-search] Chunk search failed:', e);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
export function clearChunkTextIndex() {
|
||||
chunkIdx = null;
|
||||
chunkIdxRevision = null;
|
||||
}
|
||||
@@ -1,287 +0,0 @@
|
||||
import { xbLog } from '../../../../core/debug-core.js';
|
||||
import { extensionFolderPath } from '../../../../core/constants.js';
|
||||
|
||||
const MODULE_ID = 'tokenizer';
|
||||
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
// 词性过滤
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
|
||||
// 保留的词性（名词类 + 英文）
|
||||
const KEEP_POS_PREFIXES = ['n', 'eng'];
|
||||
|
||||
function shouldKeepByPos(pos) {
|
||||
return KEEP_POS_PREFIXES.some(prefix => pos.startsWith(prefix));
|
||||
}
|
||||
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
// 语言检测
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
|
||||
function shouldUseJieba(text) {
|
||||
const zh = (text.match(/[\u4e00-\u9fff]/g) || []).length;
|
||||
return zh >= 5;
|
||||
}
|
||||
|
||||
function detectMainLanguage(text) {
|
||||
const zh = (text.match(/[\u4e00-\u9fff]/g) || []).length;
|
||||
const jp = (text.match(/[\u3040-\u309f\u30a0-\u30ff]/g) || []).length;
|
||||
const en = (text.match(/[a-zA-Z]/g) || []).length;
|
||||
const total = zh + jp + en || 1;
|
||||
|
||||
if (jp / total > 0.2) return 'jp';
|
||||
if (en / total > 0.5) return 'en';
|
||||
return 'zh';
|
||||
}
|
||||
|
||||
// 替换原有的大停用词表
|
||||
const STOP_WORDS = new Set([
|
||||
// 系统词
|
||||
'用户', '角色', '玩家', '旁白', 'user', 'assistant', 'system',
|
||||
// 时间泛词
|
||||
'时候', '现在', '今天', '明天', '昨天', '早上', '晚上',
|
||||
// 方位泛词
|
||||
'这里', '那里', '上面', '下面', '里面', '外面',
|
||||
// 泛化名词
|
||||
'东西', '事情', '事儿', '地方', '样子', '意思', '感觉',
|
||||
'一下', '一些', '一点', '一会', '一次',
|
||||
]);
|
||||
|
||||
// 英文停用词（fallback 用）
|
||||
const EN_STOP_WORDS = new Set([
|
||||
'the', 'a', 'an', 'is', 'are', 'was', 'were', 'be', 'been',
|
||||
'have', 'has', 'had', 'do', 'does', 'did', 'will', 'would',
|
||||
'could', 'should', 'may', 'might', 'must', 'can',
|
||||
'to', 'of', 'in', 'on', 'at', 'for', 'with', 'by', 'from',
|
||||
'and', 'or', 'but', 'if', 'that', 'this', 'it', 'its',
|
||||
'i', 'you', 'he', 'she', 'we', 'they',
|
||||
'my', 'your', 'his', 'her', 'our', 'their',
|
||||
'what', 'which', 'who', 'whom', 'where', 'when', 'why', 'how',
|
||||
]);
|
||||
|
||||
let jiebaModule = null;
|
||||
let jiebaReady = false;
|
||||
let jiebaLoading = false;
|
||||
|
||||
async function ensureJieba() {
|
||||
if (jiebaReady) return true;
|
||||
if (jiebaLoading) {
|
||||
for (let i = 0; i < 50; i++) {
|
||||
await new Promise(r => setTimeout(r, 100));
|
||||
if (jiebaReady) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
jiebaLoading = true;
|
||||
|
||||
try {
|
||||
const jiebaPath = `/${extensionFolderPath}/libs/jieba-wasm/jieba_rs_wasm.js`;
|
||||
// eslint-disable-next-line no-unsanitized/method
|
||||
jiebaModule = await import(jiebaPath);
|
||||
|
||||
if (jiebaModule.default) {
|
||||
await jiebaModule.default();
|
||||
}
|
||||
|
||||
jiebaReady = true;
|
||||
xbLog.info(MODULE_ID, 'jieba-wasm 加载成功');
|
||||
const keys = Object.getOwnPropertyNames(jiebaModule || {});
|
||||
const dkeys = Object.getOwnPropertyNames(jiebaModule?.default || {});
|
||||
xbLog.info(MODULE_ID, `jieba keys: ${keys.join(',')}`);
|
||||
xbLog.info(MODULE_ID, `jieba default keys: ${dkeys.join(',')}`);
|
||||
xbLog.info(MODULE_ID, `jieba.tag: ${typeof jiebaModule?.tag}`);
|
||||
return true;
|
||||
} catch (e) {
|
||||
xbLog.error(MODULE_ID, 'jieba-wasm 加载失败', e);
|
||||
jiebaLoading = false;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function fallbackTokenize(text) {
|
||||
const tokens = [];
|
||||
const lang = detectMainLanguage(text);
|
||||
|
||||
// 英文
|
||||
const enMatches = text.match(/[a-zA-Z]{2,20}/gi) || [];
|
||||
tokens.push(...enMatches.filter(w => !EN_STOP_WORDS.has(w.toLowerCase())));
|
||||
|
||||
// 日语假名
|
||||
if (lang === 'jp') {
|
||||
const kanaMatches = text.match(/[\u3040-\u309f\u30a0-\u30ff]{2,10}/g) || [];
|
||||
tokens.push(...kanaMatches);
|
||||
}
|
||||
|
||||
// 中文/日语汉字
|
||||
const zhMatches = text.match(/[\u4e00-\u9fff]{2,6}/g) || [];
|
||||
tokens.push(...zhMatches);
|
||||
|
||||
// 数字+汉字组合
|
||||
const numZhMatches = text.match(/\d+[\u4e00-\u9fff]{1,4}/g) || [];
|
||||
tokens.push(...numZhMatches);
|
||||
|
||||
return tokens;
|
||||
}
|
||||
|
||||
export async function extractNouns(text, options = {}) {
|
||||
const { minLen = 2, maxCount = 0 } = options;
|
||||
if (!text?.trim()) return [];
|
||||
|
||||
// 中文为主 → 用 jieba
|
||||
if (shouldUseJieba(text)) {
|
||||
const hasJieba = await ensureJieba();
|
||||
|
||||
if (hasJieba && jiebaModule?.tag) {
|
||||
try {
|
||||
const tagged = jiebaModule.tag(text, true);
|
||||
|
||||
const result = [];
|
||||
const seen = new Set();
|
||||
|
||||
const list = Array.isArray(tagged) ? tagged : [];
|
||||
for (const item of list) {
|
||||
let word = '';
|
||||
let pos = '';
|
||||
if (Array.isArray(item)) {
|
||||
[word, pos] = item;
|
||||
} else if (item && typeof item === 'object') {
|
||||
word = item.word || item.w || item.text || item.term || '';
|
||||
pos = item.tag || item.pos || item.p || '';
|
||||
}
|
||||
if (!word || !pos) continue;
|
||||
if (word.length < minLen) continue;
|
||||
if (!shouldKeepByPos(pos)) continue;
|
||||
if (STOP_WORDS.has(word)) continue;
|
||||
if (seen.has(word)) continue;
|
||||
|
||||
seen.add(word);
|
||||
result.push(word);
|
||||
|
||||
if (maxCount > 0 && result.length >= maxCount) break;
|
||||
}
|
||||
|
||||
return result;
|
||||
} catch (e) {
|
||||
xbLog.warn(MODULE_ID, 'jieba tag 失败：' + (e && e.message ? e.message : String(e)));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 非中文 / jieba 失败 → fallback
|
||||
const tokens = fallbackTokenize(text);
|
||||
|
||||
const result = [];
|
||||
const seen = new Set();
|
||||
|
||||
for (const t of tokens) {
|
||||
if (t.length < minLen) continue;
|
||||
if (STOP_WORDS.has(t)) continue;
|
||||
if (seen.has(t)) continue;
|
||||
|
||||
seen.add(t);
|
||||
result.push(t);
|
||||
|
||||
if (maxCount > 0 && result.length >= maxCount) break;
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
export async function extractRareTerms(text, maxCount = 15) {
|
||||
if (!text?.trim()) return [];
|
||||
|
||||
// 中文为主 → 用 jieba
|
||||
if (shouldUseJieba(text)) {
|
||||
const hasJieba = await ensureJieba();
|
||||
|
||||
if (hasJieba && jiebaModule?.tag) {
|
||||
try {
|
||||
const tagged = jiebaModule.tag(text, true);
|
||||
|
||||
const candidates = [];
|
||||
const seen = new Set();
|
||||
|
||||
const list = Array.isArray(tagged) ? tagged : [];
|
||||
for (const item of list) {
|
||||
let word = '';
|
||||
let pos = '';
|
||||
if (Array.isArray(item)) {
|
||||
[word, pos] = item;
|
||||
} else if (item && typeof item === 'object') {
|
||||
word = item.word || item.w || item.text || item.term || '';
|
||||
pos = item.tag || item.pos || item.p || '';
|
||||
}
|
||||
if (!word || !pos) continue;
|
||||
if (word.length < 2) continue;
|
||||
if (!shouldKeepByPos(pos)) continue;
|
||||
if (STOP_WORDS.has(word)) continue;
|
||||
if (seen.has(word)) continue;
|
||||
|
||||
seen.add(word);
|
||||
|
||||
// 稀有度评分
|
||||
let score = 0;
|
||||
if (word.length >= 4) score += 3;
|
||||
else if (word.length >= 3) score += 1;
|
||||
if (/[a-zA-Z]/.test(word)) score += 2;
|
||||
if (/\d/.test(word)) score += 1;
|
||||
// 专名词性加分
|
||||
if (['nr', 'ns', 'nt', 'nz'].some(p => pos.startsWith(p))) score += 2;
|
||||
|
||||
candidates.push({ term: word, score });
|
||||
}
|
||||
|
||||
candidates.sort((a, b) => b.score - a.score);
|
||||
return candidates.slice(0, maxCount).map(x => x.term);
|
||||
} catch (e) {
|
||||
xbLog.warn(MODULE_ID, 'jieba tag 失败：' + (e && e.message ? e.message : String(e)));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 非中文 / jieba 失败 → fallback
|
||||
const allNouns = await extractNouns(text, { minLen: 2, maxCount: 0 });
|
||||
|
||||
const scored = allNouns.map(t => {
|
||||
let score = 0;
|
||||
if (t.length >= 4) score += 3;
|
||||
else if (t.length >= 3) score += 1;
|
||||
if (/[a-zA-Z]/.test(t)) score += 2;
|
||||
if (/\d/.test(t)) score += 1;
|
||||
return { term: t, score };
|
||||
});
|
||||
|
||||
scored.sort((a, b) => b.score - a.score);
|
||||
return scored.slice(0, maxCount).map(x => x.term);
|
||||
}
|
||||
|
||||
export async function extractNounsFromFactsO(facts, relevantSubjects, maxCount = 5) {
|
||||
if (!facts?.length || !relevantSubjects?.size) return [];
|
||||
|
||||
const oTexts = [];
|
||||
|
||||
for (const f of facts) {
|
||||
if (f.retracted) continue;
|
||||
|
||||
// 只取相关主体的 facts
|
||||
const s = String(f.s || '').trim();
|
||||
if (!relevantSubjects.has(s)) continue;
|
||||
|
||||
const o = String(f.o || '').trim();
|
||||
if (!o) continue;
|
||||
|
||||
// 跳过太长的 O（可能是完整句子）
|
||||
if (o.length > 30) continue;
|
||||
|
||||
oTexts.push(o);
|
||||
}
|
||||
|
||||
if (!oTexts.length) return [];
|
||||
|
||||
const combined = oTexts.join(' ');
|
||||
return await extractNouns(combined, { minLen: 2, maxCount });
|
||||
}
|
||||
|
||||
export { ensureJieba };
|
||||
|
||||
@@ -35,6 +35,58 @@ function ensureStateAtomsArray() {
|
||||
return chat_metadata.extensions[EXT_ID].stateAtoms;
|
||||
}
|
||||
|
||||
// L0Index: per-floor status (ok | empty | fail)
|
||||
function ensureL0Index() {
|
||||
chat_metadata.extensions ||= {};
|
||||
chat_metadata.extensions[EXT_ID] ||= {};
|
||||
chat_metadata.extensions[EXT_ID].l0Index ||= { version: 1, byFloor: {} };
|
||||
chat_metadata.extensions[EXT_ID].l0Index.byFloor ||= {};
|
||||
return chat_metadata.extensions[EXT_ID].l0Index;
|
||||
}
|
||||
|
||||
export function getL0Index() {
|
||||
return ensureL0Index();
|
||||
}
|
||||
|
||||
export function getL0FloorStatus(floor) {
|
||||
const idx = ensureL0Index();
|
||||
return idx.byFloor?.[String(floor)] || null;
|
||||
}
|
||||
|
||||
export function setL0FloorStatus(floor, record) {
|
||||
const idx = ensureL0Index();
|
||||
idx.byFloor[String(floor)] = {
|
||||
...record,
|
||||
floor,
|
||||
updatedAt: Date.now(),
|
||||
};
|
||||
saveMetadataDebounced();
|
||||
}
|
||||
|
||||
export function clearL0Index() {
|
||||
const idx = ensureL0Index();
|
||||
idx.byFloor = {};
|
||||
saveMetadataDebounced();
|
||||
}
|
||||
|
||||
export function deleteL0IndexFromFloor(fromFloor) {
|
||||
const idx = ensureL0Index();
|
||||
const keys = Object.keys(idx.byFloor || {});
|
||||
let deleted = 0;
|
||||
for (const k of keys) {
|
||||
const f = Number(k);
|
||||
if (Number.isFinite(f) && f >= fromFloor) {
|
||||
delete idx.byFloor[k];
|
||||
deleted++;
|
||||
}
|
||||
}
|
||||
if (deleted > 0) {
|
||||
saveMetadataDebounced();
|
||||
xbLog.info(MODULE_ID, `删除 ${deleted} 条 L0Index (floor >= ${fromFloor})`);
|
||||
}
|
||||
return deleted;
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取当前聊天的所有 StateAtoms
|
||||
*/
|
||||
@@ -113,6 +165,30 @@ export function getStateAtomsCount() {
|
||||
return ensureStateAtomsArray().length;
|
||||
}
|
||||
|
||||
/**
|
||||
* Return floors that already have extracted atoms.
|
||||
*/
|
||||
export function getExtractedFloors() {
|
||||
const floors = new Set();
|
||||
const arr = ensureStateAtomsArray();
|
||||
for (const atom of arr) {
|
||||
if (typeof atom?.floor === 'number' && atom.floor >= 0) {
|
||||
floors.add(atom.floor);
|
||||
}
|
||||
}
|
||||
return floors;
|
||||
}
|
||||
|
||||
/**
|
||||
* Replace all stored StateAtoms.
|
||||
*/
|
||||
export function replaceStateAtoms(atoms) {
|
||||
const next = Array.isArray(atoms) ? atoms : [];
|
||||
chat_metadata.extensions[EXT_ID].stateAtoms = next;
|
||||
saveMetadataDebounced();
|
||||
xbLog.info(MODULE_ID, `替换 StateAtoms: ${next.length} 条`);
|
||||
}
|
||||
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
// StateVector 操作（IndexedDB）
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
|
||||
@@ -1,648 +1,83 @@
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
// Story Summary - Embedding Service
|
||||
// 统一的向量生成接口（本地模型 / 在线服务）
|
||||
// Story Summary - Embedder (v2 - 统一硅基)
|
||||
// 所有 embedding 请求转发到 siliconflow.js
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
|
||||
import { xbLog } from '../../../../core/debug-core.js';
|
||||
|
||||
const MODULE_ID = 'embedding';
|
||||
|
||||
import { embed as sfEmbed, getApiKey } from '../llm/siliconflow.js';
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
// 本地模型配置
|
||||
// 统一 embed 接口
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
|
||||
export const LOCAL_MODELS = {
|
||||
'bge-small-zh': {
|
||||
id: 'bge-small-zh',
|
||||
name: '中文轻量 (51MB)',
|
||||
hfId: 'Xenova/bge-small-zh-v1.5',
|
||||
dims: 512,
|
||||
desc: '手机/低配适用',
|
||||
},
|
||||
'bge-base-zh': {
|
||||
id: 'bge-base-zh',
|
||||
name: '中文标准 (102MB)',
|
||||
hfId: 'Xenova/bge-base-zh-v1.5',
|
||||
dims: 768,
|
||||
desc: 'PC 推荐，效果更好',
|
||||
},
|
||||
'e5-small': {
|
||||
id: 'e5-small',
|
||||
name: '多语言 (118MB)',
|
||||
hfId: 'Xenova/multilingual-e5-small',
|
||||
dims: 384,
|
||||
desc: '非中文用户',
|
||||
},
|
||||
};
|
||||
|
||||
export const DEFAULT_LOCAL_MODEL = 'bge-small-zh';
|
||||
export async function embed(texts, config, options = {}) {
|
||||
// 忽略旧的 config 参数，统一走硅基
|
||||
return await sfEmbed(texts, options);
|
||||
}
|
||||
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
// 在线服务配置
|
||||
// 指纹（简化版）
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
|
||||
export function getEngineFingerprint(config) {
|
||||
// 统一使用硅基 bge-m3
|
||||
return 'siliconflow:bge-m3:1024';
|
||||
}
|
||||
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
// 状态检查（简化版）
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
|
||||
export async function checkLocalModelStatus() {
|
||||
// 不再支持本地模型
|
||||
return { status: 'not_supported', message: '请使用在线服务' };
|
||||
}
|
||||
|
||||
export function isLocalModelLoaded() {
|
||||
return false;
|
||||
}
|
||||
|
||||
export async function downloadLocalModel() {
|
||||
throw new Error('本地模型已移除，请使用在线服务');
|
||||
}
|
||||
|
||||
export function cancelDownload() { }
|
||||
|
||||
export async function deleteLocalModelCache() { }
|
||||
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
// 在线服务测试
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
|
||||
export async function testOnlineService() {
|
||||
const key = getApiKey();
|
||||
if (!key) {
|
||||
throw new Error('请配置硅基 API Key');
|
||||
}
|
||||
|
||||
try {
|
||||
const [vec] = await sfEmbed(['测试连接']);
|
||||
return { success: true, dims: vec?.length || 0 };
|
||||
} catch (e) {
|
||||
throw new Error(`连接失败: ${e.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
export async function fetchOnlineModels() {
|
||||
// 硅基模型固定
|
||||
return ['BAAI/bge-m3'];
|
||||
}
|
||||
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
// 兼容旧接口
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
|
||||
export const DEFAULT_LOCAL_MODEL = 'bge-m3';
|
||||
|
||||
export const LOCAL_MODELS = {};
|
||||
|
||||
export const ONLINE_PROVIDERS = {
|
||||
siliconflow: {
|
||||
id: 'siliconflow',
|
||||
name: '硅基流动',
|
||||
baseUrl: 'https://api.siliconflow.cn',
|
||||
canFetchModels: false,
|
||||
defaultModels: [
|
||||
'BAAI/bge-m3',
|
||||
'BAAI/bge-large-zh-v1.5',
|
||||
'BAAI/bge-small-zh-v1.5',
|
||||
],
|
||||
},
|
||||
cohere: {
|
||||
id: 'cohere',
|
||||
name: 'Cohere',
|
||||
baseUrl: 'https://api.cohere.ai',
|
||||
canFetchModels: false,
|
||||
defaultModels: [
|
||||
'embed-multilingual-v3.0',
|
||||
'embed-english-v3.0',
|
||||
],
|
||||
// Cohere 使用不同的 API 格式
|
||||
customEmbed: true,
|
||||
},
|
||||
openai: {
|
||||
id: 'openai',
|
||||
name: 'OpenAI 兼容',
|
||||
baseUrl: '',
|
||||
canFetchModels: true,
|
||||
defaultModels: [],
|
||||
},
|
||||
};
|
||||
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
// 本地模型状态管理
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
|
||||
// 已加载的模型实例：{ modelId: pipeline }
|
||||
const loadedPipelines = {};
|
||||
|
||||
// 当前正在下载的模型
|
||||
let downloadingModelId = null;
|
||||
let downloadAbortController = null;
|
||||
|
||||
// Worker for local embedding
|
||||
let embeddingWorker = null;
|
||||
let workerRequestId = 0;
|
||||
const workerCallbacks = new Map();
|
||||
|
||||
function getWorker() {
|
||||
if (!embeddingWorker) {
|
||||
const workerPath = new URL('./embedder.worker.js', import.meta.url).href;
|
||||
embeddingWorker = new Worker(workerPath, { type: 'module' });
|
||||
|
||||
embeddingWorker.onmessage = (e) => {
|
||||
const { requestId, ...data } = e.data || {};
|
||||
const callback = workerCallbacks.get(requestId);
|
||||
if (callback) {
|
||||
callback(data);
|
||||
if (data.type === 'result' || data.type === 'error' || data.type === 'loaded') {
|
||||
workerCallbacks.delete(requestId);
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
return embeddingWorker;
|
||||
}
|
||||
|
||||
function workerRequest(message) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const requestId = ++workerRequestId;
|
||||
const worker = getWorker();
|
||||
|
||||
workerCallbacks.set(requestId, (data) => {
|
||||
if (data.type === 'error') {
|
||||
reject(new Error(data.error));
|
||||
} else if (data.type === 'result') {
|
||||
resolve(data.vectors);
|
||||
} else if (data.type === 'loaded') {
|
||||
resolve(true);
|
||||
}
|
||||
});
|
||||
|
||||
worker.postMessage({ ...message, requestId });
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
// 本地模型操作
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
|
||||
/**
|
||||
* 检查指定本地模型的状态
|
||||
* 只读取缓存，绝不触发下载
|
||||
*/
|
||||
export async function checkLocalModelStatus(modelId = DEFAULT_LOCAL_MODEL) {
|
||||
const modelConfig = LOCAL_MODELS[modelId];
|
||||
if (!modelConfig) {
|
||||
return { status: 'error', message: '未知模型' };
|
||||
}
|
||||
|
||||
// 已加载到内存
|
||||
if (loadedPipelines[modelId]) {
|
||||
return { status: 'ready', message: '已就绪' };
|
||||
}
|
||||
|
||||
// 正在下载
|
||||
if (downloadingModelId === modelId) {
|
||||
return { status: 'downloading', message: '下载中' };
|
||||
}
|
||||
|
||||
// 检查 IndexedDB 缓存
|
||||
const hasCache = await checkModelCache(modelConfig.hfId);
|
||||
if (hasCache) {
|
||||
return { status: 'cached', message: '已缓存，可加载' };
|
||||
}
|
||||
|
||||
return { status: 'not_downloaded', message: '未下载' };
|
||||
}
|
||||
|
||||
/**
|
||||
* 检查 IndexedDB 中是否有模型缓存
|
||||
*/
|
||||
async function checkModelCache(hfId) {
|
||||
return new Promise((resolve) => {
|
||||
try {
|
||||
const request = indexedDB.open('transformers-cache', 1);
|
||||
request.onerror = () => resolve(false);
|
||||
request.onsuccess = (event) => {
|
||||
const db = event.target.result;
|
||||
const storeNames = Array.from(db.objectStoreNames);
|
||||
db.close();
|
||||
// 检查是否有该模型的缓存
|
||||
const modelKey = hfId.replace('/', '_');
|
||||
const hasModel = storeNames.some(name =>
|
||||
name.includes(modelKey) || name.includes('onnx')
|
||||
);
|
||||
resolve(hasModel);
|
||||
};
|
||||
request.onupgradeneeded = () => resolve(false);
|
||||
} catch {
|
||||
resolve(false);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* 下载/加载本地模型
|
||||
* @param {string} modelId - 模型ID
|
||||
* @param {Function} onProgress - 进度回调 (0-100)
|
||||
* @returns {Promise<boolean>}
|
||||
*/
|
||||
export async function downloadLocalModel(modelId = DEFAULT_LOCAL_MODEL, onProgress) {
|
||||
const modelConfig = LOCAL_MODELS[modelId];
|
||||
if (!modelConfig) {
|
||||
throw new Error(`未知模型: ${modelId}`);
|
||||
}
|
||||
// 已加载
|
||||
if (loadedPipelines[modelId]) {
|
||||
onProgress?.(100);
|
||||
return true;
|
||||
}
|
||||
// 正在下载其他模型
|
||||
if (downloadingModelId && downloadingModelId !== modelId) {
|
||||
throw new Error(`正在下载其他模型: ${downloadingModelId}`);
|
||||
}
|
||||
// 正在下载同一模型，等待完成
|
||||
if (downloadingModelId === modelId) {
|
||||
xbLog.info(MODULE_ID, `模型 ${modelId} 正在加载中...`);
|
||||
return new Promise((resolve, reject) => {
|
||||
const check = () => {
|
||||
if (loadedPipelines[modelId]) {
|
||||
resolve(true);
|
||||
} else if (downloadingModelId !== modelId) {
|
||||
reject(new Error('下载已取消'));
|
||||
} else {
|
||||
setTimeout(check, 200);
|
||||
}
|
||||
};
|
||||
check();
|
||||
});
|
||||
}
|
||||
downloadingModelId = modelId;
|
||||
downloadAbortController = new AbortController();
|
||||
|
||||
try {
|
||||
xbLog.info(MODULE_ID, `开始下载模型: ${modelId}`);
|
||||
|
||||
return await new Promise((resolve, reject) => {
|
||||
const requestId = ++workerRequestId;
|
||||
const worker = getWorker();
|
||||
|
||||
workerCallbacks.set(requestId, (data) => {
|
||||
if (data.type === 'progress') {
|
||||
onProgress?.(data.percent);
|
||||
} else if (data.type === 'loaded') {
|
||||
loadedPipelines[modelId] = true;
|
||||
workerCallbacks.delete(requestId);
|
||||
resolve(true);
|
||||
} else if (data.type === 'error') {
|
||||
workerCallbacks.delete(requestId);
|
||||
reject(new Error(data.error));
|
||||
}
|
||||
});
|
||||
|
||||
worker.postMessage({
|
||||
type: 'load',
|
||||
modelId,
|
||||
hfId: modelConfig.hfId,
|
||||
requestId
|
||||
});
|
||||
});
|
||||
} finally {
|
||||
downloadingModelId = null;
|
||||
downloadAbortController = null;
|
||||
}
|
||||
}
|
||||
|
||||
export function cancelDownload() {
|
||||
if (downloadAbortController) {
|
||||
downloadAbortController.abort();
|
||||
xbLog.info(MODULE_ID, '下载已取消');
|
||||
}
|
||||
downloadingModelId = null;
|
||||
downloadAbortController = null;
|
||||
}
|
||||
|
||||
/**
|
||||
* 删除指定模型的缓存
|
||||
*/
|
||||
export async function deleteLocalModelCache(modelId = null) {
|
||||
try {
|
||||
// 删除 IndexedDB
|
||||
await new Promise((resolve, reject) => {
|
||||
const request = indexedDB.deleteDatabase('transformers-cache');
|
||||
request.onsuccess = () => resolve();
|
||||
request.onerror = () => reject(request.error);
|
||||
request.onblocked = () => {
|
||||
xbLog.warn(MODULE_ID, 'IndexedDB 删除被阻塞');
|
||||
resolve();
|
||||
};
|
||||
});
|
||||
|
||||
// 删除 CacheStorage
|
||||
if (window.caches) {
|
||||
const cacheNames = await window.caches.keys();
|
||||
for (const name of cacheNames) {
|
||||
if (name.includes('transformers') || name.includes('huggingface') || name.includes('xenova')) {
|
||||
await window.caches.delete(name);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 清除内存中的 pipeline
|
||||
if (modelId && loadedPipelines[modelId]) {
|
||||
delete loadedPipelines[modelId];
|
||||
} else {
|
||||
Object.keys(loadedPipelines).forEach(key => delete loadedPipelines[key]);
|
||||
}
|
||||
|
||||
xbLog.info(MODULE_ID, '模型缓存已清除');
|
||||
return true;
|
||||
} catch (e) {
|
||||
xbLog.error(MODULE_ID, '清除缓存失败', e);
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 使用本地模型生成向量
|
||||
*/
|
||||
async function embedLocal(texts, modelId = DEFAULT_LOCAL_MODEL) {
|
||||
if (!loadedPipelines[modelId]) {
|
||||
await downloadLocalModel(modelId);
|
||||
}
|
||||
|
||||
return await workerRequest({ type: 'embed', texts });
|
||||
}
|
||||
|
||||
export function isLocalModelLoaded(modelId = DEFAULT_LOCAL_MODEL) {
|
||||
return !!loadedPipelines[modelId];
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取本地模型信息
|
||||
*/
|
||||
export function getLocalModelInfo(modelId = DEFAULT_LOCAL_MODEL) {
|
||||
return LOCAL_MODELS[modelId] || null;
|
||||
}
|
||||
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
// 在线服务操作
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
|
||||
/**
|
||||
* 测试在线服务连接
|
||||
*/
|
||||
export async function testOnlineService(provider, config) {
|
||||
const { url, key, model } = config;
|
||||
|
||||
if (!key) {
|
||||
throw new Error('请填写 API Key');
|
||||
}
|
||||
if (!model) {
|
||||
throw new Error('请选择模型');
|
||||
}
|
||||
|
||||
const providerConfig = ONLINE_PROVIDERS[provider];
|
||||
const baseUrl = (providerConfig?.baseUrl || url || '').replace(/\/+$/, '');
|
||||
|
||||
if (!baseUrl) {
|
||||
throw new Error('请填写 API URL');
|
||||
}
|
||||
|
||||
try {
|
||||
if (provider === 'cohere') {
|
||||
// Cohere 使用不同的 API 格式
|
||||
const response = await fetch(`${baseUrl}/v1/embed`, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Authorization': `Bearer ${key}`,
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify({
|
||||
model: model,
|
||||
texts: ['测试连接'],
|
||||
input_type: 'search_document',
|
||||
}),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const error = await response.text();
|
||||
throw new Error(`API 返回 ${response.status}: ${error}`);
|
||||
}
|
||||
|
||||
const data = await response.json();
|
||||
const dims = data.embeddings?.[0]?.length || 0;
|
||||
|
||||
if (dims === 0) {
|
||||
throw new Error('API 返回的向量维度为 0');
|
||||
}
|
||||
|
||||
return { success: true, dims };
|
||||
|
||||
} else {
|
||||
// OpenAI 兼容格式
|
||||
const response = await fetch(`${baseUrl}/v1/embeddings`, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Authorization': `Bearer ${key}`,
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify({
|
||||
model: model,
|
||||
input: ['测试连接'],
|
||||
}),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const error = await response.text();
|
||||
throw new Error(`API 返回 ${response.status}: ${error}`);
|
||||
}
|
||||
|
||||
const data = await response.json();
|
||||
const dims = data.data?.[0]?.embedding?.length || 0;
|
||||
|
||||
if (dims === 0) {
|
||||
throw new Error('API 返回的向量维度为 0');
|
||||
}
|
||||
|
||||
return { success: true, dims };
|
||||
}
|
||||
|
||||
} catch (e) {
|
||||
if (e.name === 'TypeError' && e.message.includes('fetch')) {
|
||||
throw new Error('网络错误，请检查 URL 是否正确');
|
||||
}
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 拉取在线模型列表（仅 OpenAI 兼容）
|
||||
*/
|
||||
export async function fetchOnlineModels(config) {
|
||||
const { url, key } = config;
|
||||
|
||||
if (!url || !key) {
|
||||
throw new Error('请填写 URL 和 Key');
|
||||
}
|
||||
|
||||
const baseUrl = url.replace(/\/+$/, '').replace(/\/v1$/, '');
|
||||
|
||||
const response = await fetch(`${baseUrl}/v1/models`, {
|
||||
headers: {
|
||||
'Authorization': `Bearer ${key}`,
|
||||
'Accept': 'application/json',
|
||||
},
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`获取模型列表失败: ${response.status}`);
|
||||
}
|
||||
|
||||
const data = await response.json();
|
||||
const models = data.data?.map(m => m.id).filter(Boolean) || [];
|
||||
|
||||
// 过滤出 embedding 相关的模型
|
||||
const embeddingModels = models.filter(m => {
|
||||
const lower = m.toLowerCase();
|
||||
return lower.includes('embed') ||
|
||||
lower.includes('bge') ||
|
||||
lower.includes('e5') ||
|
||||
lower.includes('gte');
|
||||
});
|
||||
|
||||
return embeddingModels.length > 0 ? embeddingModels : models.slice(0, 20);
|
||||
}
|
||||
|
||||
/**
|
||||
* 使用在线服务生成向量
|
||||
*/
|
||||
async function embedOnline(texts, provider, config, options = {}) {
|
||||
const { url, key, model } = config;
|
||||
const signal = options?.signal;
|
||||
|
||||
const providerConfig = ONLINE_PROVIDERS[provider];
|
||||
const baseUrl = (providerConfig?.baseUrl || url || '').replace(/\/+$/, '');
|
||||
|
||||
// 永远重试：指数退避 + 上限 + 抖动
|
||||
const BASE_WAIT_MS = 1200;
|
||||
const MAX_WAIT_MS = 15000;
|
||||
|
||||
const sleepAbortable = (ms) => new Promise((resolve, reject) => {
|
||||
if (signal?.aborted) return reject(new DOMException('Aborted', 'AbortError'));
|
||||
const t = setTimeout(resolve, ms);
|
||||
if (signal) {
|
||||
signal.addEventListener('abort', () => {
|
||||
clearTimeout(t);
|
||||
reject(new DOMException('Aborted', 'AbortError'));
|
||||
}, { once: true });
|
||||
}
|
||||
});
|
||||
|
||||
let attempt = 0;
|
||||
while (true) {
|
||||
attempt++;
|
||||
try {
|
||||
let response;
|
||||
|
||||
if (provider === 'cohere') {
|
||||
response = await fetch(`${baseUrl}/v1/embed`, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Authorization': `Bearer ${key}`,
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify({
|
||||
model: model,
|
||||
texts: texts,
|
||||
input_type: 'search_document',
|
||||
}),
|
||||
signal,
|
||||
});
|
||||
} else {
|
||||
response = await fetch(`${baseUrl}/v1/embeddings`, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Authorization': `Bearer ${key}`,
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify({
|
||||
model: model,
|
||||
input: texts,
|
||||
}),
|
||||
signal,
|
||||
});
|
||||
}
|
||||
|
||||
// 需要“永远重试”的典型状态：
|
||||
// - 429：限流
|
||||
// - 403：配额/风控/未实名等（你提到的硅基未认证）
|
||||
// - 5xx：服务端错误
|
||||
const retryableStatus = (s) => s === 429 || s === 403 || (s >= 500 && s <= 599);
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text().catch(() => '');
|
||||
|
||||
if (retryableStatus(response.status)) {
|
||||
const exp = Math.min(MAX_WAIT_MS, BASE_WAIT_MS * Math.pow(2, Math.min(attempt, 6) - 1));
|
||||
const jitter = Math.floor(Math.random() * 350);
|
||||
const waitMs = exp + jitter;
|
||||
await sleepAbortable(waitMs);
|
||||
continue;
|
||||
}
|
||||
|
||||
// 非可恢复错误：直接抛出（比如 400 参数错、401 key 错等）
|
||||
const err = new Error(`API 返回 ${response.status}: ${errorText}`);
|
||||
err.status = response.status;
|
||||
throw err;
|
||||
}
|
||||
|
||||
const data = await response.json();
|
||||
|
||||
if (provider === 'cohere') {
|
||||
return (data.embeddings || []).map(e => Array.isArray(e) ? e : Array.from(e));
|
||||
}
|
||||
return (data.data || []).map(item => {
|
||||
const embedding = item.embedding;
|
||||
return Array.isArray(embedding) ? embedding : Array.from(embedding);
|
||||
});
|
||||
} catch (e) {
|
||||
// 取消：必须立刻退出
|
||||
if (e?.name === 'AbortError') throw e;
|
||||
|
||||
// 网络错误：永远重试
|
||||
const exp = Math.min(MAX_WAIT_MS, BASE_WAIT_MS * Math.pow(2, Math.min(attempt, 6) - 1));
|
||||
const jitter = Math.floor(Math.random() * 350);
|
||||
const waitMs = exp + jitter;
|
||||
await sleepAbortable(waitMs);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
// 统一接口
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
|
||||
/**
|
||||
* 生成向量（统一接口）
|
||||
* @param {string[]} texts - 要向量化的文本数组
|
||||
* @param {Object} config - 配置
|
||||
* @returns {Promise<number[][]>}
|
||||
*/
|
||||
export async function embed(texts, config, options = {}) {
|
||||
if (!texts?.length) return [];
|
||||
|
||||
const { engine, local, online } = config;
|
||||
|
||||
if (engine === 'local') {
|
||||
const modelId = local?.modelId || DEFAULT_LOCAL_MODEL;
|
||||
return await embedLocal(texts, modelId);
|
||||
|
||||
} else if (engine === 'online') {
|
||||
const provider = online?.provider || 'siliconflow';
|
||||
if (!online?.key || !online?.model) {
|
||||
throw new Error('在线服务配置不完整');
|
||||
}
|
||||
return await embedOnline(texts, provider, online, options);
|
||||
|
||||
} else {
|
||||
throw new Error(`未知的引擎类型: ${engine}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取当前引擎的唯一标识（用于检查向量是否匹配）
|
||||
*/
|
||||
|
||||
// Concurrent embed for online services (local falls back to sequential)
|
||||
export async function embedBatchesConcurrent(textBatches, config, concurrency = 3) {
|
||||
if (config.engine === 'local' || textBatches.length <= 1) {
|
||||
const results = [];
|
||||
for (const batch of textBatches) {
|
||||
results.push(await embed(batch, config));
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
const results = new Array(textBatches.length);
|
||||
let index = 0;
|
||||
|
||||
async function worker() {
|
||||
while (index < textBatches.length) {
|
||||
const i = index++;
|
||||
results[i] = await embed(textBatches[i], config);
|
||||
}
|
||||
}
|
||||
|
||||
await Promise.all(
|
||||
Array(Math.min(concurrency, textBatches.length))
|
||||
.fill(null)
|
||||
.map(() => worker())
|
||||
);
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
export function getEngineFingerprint(config) {
|
||||
if (config.engine === 'local') {
|
||||
const modelId = config.local?.modelId || DEFAULT_LOCAL_MODEL;
|
||||
const modelConfig = LOCAL_MODELS[modelId];
|
||||
return `local:${modelId}:${modelConfig?.dims || 512}`;
|
||||
|
||||
} else if (config.engine === 'online') {
|
||||
const provider = config.online?.provider || 'unknown';
|
||||
const model = config.online?.model || 'unknown';
|
||||
return `online:${provider}:${model}`;
|
||||
|
||||
} else {
|
||||
return 'unknown';
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user