X/LittleWhiteBox
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add vector IO and text filtering

Browse Source
This commit is contained in:
bielie
2026-01-29 17:02:51 +08:00
parent fc23781e17
commit ee5f02fff9
10 changed files with 3368 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files

136
modules/story-summary/story-summary.js
View File

@@ -58,14 +58,12 @@ import {
import {
getMeta,
updateMeta,
getAllEventVectors,
saveEventVectors as saveEventVectorsToDb,
clearEventVectors,
clearAllChunks,
saveChunks,
saveChunkVectors,
getStorageStats,
ensureFingerprintMatch,
} from "./vector/chunk-store.js";
import {
@@ -77,6 +75,9 @@ import {
syncOnMessageReceived,
} from "./vector/chunk-builder.js";
// vector io
import { exportVectors, importVectors } from "./vector/vector-io.js";
// ═══════════════════════════════════════════════════════════════════════════
// 常量
// ═══════════════════════════════════════════════════════════════════════════
@@ -110,6 +111,10 @@ const HIDE_APPLY_DEBOUNCE_MS = 250;
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
// 向量提醒节流
let lastVectorWarningAt = 0;
const VECTOR_WARNING_COOLDOWN_MS = 120000; // 2分钟内不重复提醒
const EXT_PROMPT_KEY = "LittleWhiteBox_StorySummary";
// ═══════════════════════════════════════════════════════════════════════════
@@ -334,7 +339,7 @@ async function handleGenerateVectors(vectorCfg) {
const fingerprint = getEngineFingerprint(vectorCfg);
const isLocal = vectorCfg.engine === "local";
const batchSize = isLocal ? 5 : 20;
const batchSize = isLocal ? 5 : 25;
const concurrency = isLocal ? 1 : 2;
await clearAllChunks(chatId);
@@ -363,12 +368,10 @@ async function handleGenerateVectors(vectorCfg) {
const store = getSummaryStore();
const events = store?.json?.events || [];
await ensureFingerprintMatch(chatId, fingerprint);
const existingVectors = await getAllEventVectors(chatId);
const existingIds = new Set(existingVectors.map((v) => v.eventId));
// L2: 全量重建（先清空再重建，保持与 L1 一致性）
await clearEventVectors(chatId);
const l2Pairs = events
.filter((e) => !existingIds.has(e.id))
.map((e) => ({ id: e.id, text: `${e.title || ""} ${e.summary || ""}`.trim() }))
.filter((p) => p.text);
@@ -386,7 +389,7 @@ async function handleGenerateVectors(vectorCfg) {
const l1Total = allChunks.length;
const l2Total = events.length;
let l1Completed = 0;
let l2Completed = existingIds.size;
let l2Completed = 0;
postToFrame({ type: "VECTOR_GEN_PROGRESS", phase: "L1", current: 0, total: l1Total });
postToFrame({ type: "VECTOR_GEN_PROGRESS", phase: "L2", current: l2Completed, total: l2Total });
@@ -482,6 +485,9 @@ async function handleGenerateVectors(vectorCfg) {
await saveEventVectorsToDb(chatId, l2VectorItems, fingerprint);
}
// 更新 fingerprint（无论之前是否匹配）
await updateMeta(chatId, { fingerprint });
postToFrame({ type: "VECTOR_GEN_PROGRESS", phase: "L1", current: -1, total: 0 });
postToFrame({ type: "VECTOR_GEN_PROGRESS", phase: "L2", current: -1, total: 0 });
await sendVectorStatsToFrame();
@@ -493,6 +499,57 @@ async function handleGenerateVectors(vectorCfg) {
xbLog.info(MODULE_ID, `向量生成完成: L1=${l1Vectors.filter(Boolean).length}, L2=${l2VectorItems.length}`);
}
// ═══════════════════════════════════════════════════════════════════════════
// 向量完整性检测（仅提醒，不自动操作）
// ═══════════════════════════════════════════════════════════════════════════
async function checkVectorIntegrityAndWarn() {
const vectorCfg = getVectorConfig();
if (!vectorCfg?.enabled) return;
// 节流：2分钟内不重复提醒
const now = Date.now();
if (now - lastVectorWarningAt < VECTOR_WARNING_COOLDOWN_MS) return;
const { chat, chatId } = getContext();
if (!chatId || !chat?.length) return;
const store = getSummaryStore();
const totalFloors = chat.length;
const totalEvents = store?.json?.events?.length || 0;
// 如果没有总结数据，不需要向量
if (totalEvents === 0) return;
const meta = await getMeta(chatId);
const stats = await getStorageStats(chatId);
const fingerprint = getEngineFingerprint(vectorCfg);
const issues = [];
// 指纹不匹配
if (meta.fingerprint && meta.fingerprint !== fingerprint) {
issues.push('向量引擎/模型已变更');
}
// L1 不完整
const chunkFloorGap = totalFloors - 1 - (meta.lastChunkFloor ?? -1);
if (chunkFloorGap > 0) {
issues.push(`${chunkFloorGap} 层片段未向量化`);
}
// L2 不完整
const eventVectorGap = totalEvents - stats.eventVectors;
if (eventVectorGap > 0) {
issues.push(`${eventVectorGap} 个事件未向量化`);
}
if (issues.length > 0) {
lastVectorWarningAt = now;
await executeSlashCommand(`/echo severity=warning 向量数据不完整：${issues.join('、')}。请打开剧情总结面板点击"生成向量"。`);
}
}
async function handleClearVectors() {
const { chatId } = getContext();
if (!chatId) return;
@@ -918,6 +975,66 @@ function handleFrameMessage(event) {
try { vectorAbortController?.abort?.(); } catch {}
break;
case "VECTOR_EXPORT":
(async () => {
try {
const result = await exportVectors((status) => {
postToFrame({ type: "VECTOR_IO_STATUS", status });
});
postToFrame({
type: "VECTOR_EXPORT_RESULT",
success: true,
filename: result.filename,
size: result.size,
chunkCount: result.chunkCount,
eventCount: result.eventCount,
});
} catch (e) {
postToFrame({ type: "VECTOR_EXPORT_RESULT", success: false, error: e.message });
}
})();
break;
case "VECTOR_IMPORT_PICK":
// 在 parent 创建 file picker，避免 iframe 传大文件
(async () => {
const input = document.createElement("input");
input.type = "file";
input.accept = ".zip";
input.onchange = async () => {
const file = input.files?.[0];
if (!file) {
postToFrame({ type: "VECTOR_IMPORT_RESULT", success: false, error: "未选择文件" });
return;
}
try {
const result = await importVectors(file, (status) => {
postToFrame({ type: "VECTOR_IO_STATUS", status });
});
postToFrame({
type: "VECTOR_IMPORT_RESULT",
success: true,
chunkCount: result.chunkCount,
eventCount: result.eventCount,
warnings: result.warnings,
fingerprintMismatch: result.fingerprintMismatch,
});
await sendVectorStatsToFrame();
} catch (e) {
postToFrame({ type: "VECTOR_IMPORT_RESULT", success: false, error: e.message });
}
};
input.click();
})();
break;
case "REQUEST_VECTOR_STATS":
sendVectorStatsToFrame();
break;
case "REQUEST_CLEAR": {
const { chat, chatId } = getContext();
clearSummaryData(chatId);
@@ -1051,6 +1168,9 @@ async function handleChatChanged() {
await sendFrameBaseData(store, newLength);
sendFrameFullData(store, newLength);
}
// 检测向量完整性并提醒（仅提醒，不自动操作）
setTimeout(() => checkVectorIntegrityAndWarn(), 2000);
}
async function handleMessageDeleted() {