Fix vector build and lexical index updates

This commit is contained in:
2026-02-11 13:54:29 +08:00
parent 6c6091a942
commit 8d062d39b5

View File

@@ -1,4 +1,4 @@
// ═══════════════════════════════════════════════════════════════════════════ // ═══════════════════════════════════════════════════════════════════════════
// Story Summary - 主入口 // Story Summary - 主入口
// //
// 稳定目标: // 稳定目标:
@@ -91,7 +91,7 @@ import {
// vector io // vector io
import { exportVectors, importVectors } from "./vector/storage/vector-io.js"; import { exportVectors, importVectors } from "./vector/storage/vector-io.js";
import { invalidateLexicalIndex, warmupIndex } from "./vector/retrieval/lexical-index.js"; import { invalidateLexicalIndex, warmupIndex, addDocumentsForFloor, removeDocumentsByFloor, addEventDocuments } from "./vector/retrieval/lexical-index.js";
// ═══════════════════════════════════════════════════════════════════════════ // ═══════════════════════════════════════════════════════════════════════════
// 常量 // 常量
@@ -351,9 +351,12 @@ async function handleAnchorGenerate() {
}); });
postToFrame({ type: "ANCHOR_GEN_PROGRESS", current: 0, total: 1, message: "向量化 L1..." }); postToFrame({ type: "ANCHOR_GEN_PROGRESS", current: 0, total: 1, message: "向量化 L1..." });
await buildIncrementalChunks({ vectorConfig: vectorCfg }); const chunkResult = await buildIncrementalChunks({ vectorConfig: vectorCfg });
invalidateLexicalIndex(); // L1 rebuild only if new chunks were added (usually 0 in normal chat)
if (chunkResult.built > 0) {
invalidateLexicalIndex();
}
await sendAnchorStatsToFrame(); await sendAnchorStatsToFrame();
await sendVectorStatsToFrame(); await sendVectorStatsToFrame();
@@ -589,8 +592,6 @@ function refreshEntityLexiconAndWarmup() {
injectEntities(lexicon, displayMap); injectEntities(lexicon, displayMap);
// 异步预建词法索引(不阻塞) // 异步预建词法索引(不阻塞)
invalidateLexicalIndex();
warmupIndex();
} }
// ═══════════════════════════════════════════════════════════════════════════ // ═══════════════════════════════════════════════════════════════════════════
@@ -615,9 +616,12 @@ async function maybeAutoExtractL0() {
await incrementalExtractAtoms(chatId, chat, null, { maxFloors: 20 }); await incrementalExtractAtoms(chatId, chat, null, { maxFloors: 20 });
// 为新提取的 L0 楼层构建 L1 chunks // 为新提取的 L0 楼层构建 L1 chunks
await buildIncrementalChunks({ vectorConfig: vectorCfg }); const chunkResult = await buildIncrementalChunks({ vectorConfig: vectorCfg });
invalidateLexicalIndex(); // L1 rebuild only if new chunks were added
if (chunkResult.built > 0) {
invalidateLexicalIndex();
}
await sendAnchorStatsToFrame(); await sendAnchorStatsToFrame();
await sendVectorStatsToFrame(); await sendVectorStatsToFrame();
@@ -637,7 +641,7 @@ async function maybeAutoExtractL0() {
function warmupEmbeddingConnection() { function warmupEmbeddingConnection() {
const vectorCfg = getVectorConfig(); const vectorCfg = getVectorConfig();
if (!vectorCfg?.enabled) return; if (!vectorCfg?.enabled) return;
embed(['.'], vectorCfg, { timeout: 5000 }).catch(() => {}); embed(['.'], vectorCfg, { timeout: 5000 }).catch(() => { });
} }
async function autoVectorizeNewEvents(newEventIds) { async function autoVectorizeNewEvents(newEventIds) {
@@ -1043,7 +1047,12 @@ async function autoRunSummaryWithRetry(targetMesId, configForRun) {
const store = getSummaryStore(); const store = getSummaryStore();
postToFrame({ type: "SUMMARY_FULL_DATA", payload: buildFramePayload(store) }); postToFrame({ type: "SUMMARY_FULL_DATA", payload: buildFramePayload(store) });
invalidateLexicalIndex(); // Incrementally add new events to the lexical index
if (newEventIds?.length) {
const allEvents = store?.json?.events || [];
const idSet = new Set(newEventIds);
addEventDocuments(allEvents.filter(e => idSet.has(e.id)));
}
applyHideStateDebounced(); applyHideStateDebounced();
updateFrameStatsAfterSummary(endMesId, store.json || {}); updateFrameStatsAfterSummary(endMesId, store.json || {});
@@ -1150,7 +1159,7 @@ function handleFrameMessage(event) {
case "VECTOR_CANCEL_GENERATE": case "VECTOR_CANCEL_GENERATE":
vectorCancelled = true; vectorCancelled = true;
cancelL0Extraction(); cancelL0Extraction();
try { vectorAbortController?.abort?.(); } catch {} try { vectorAbortController?.abort?.(); } catch { }
postToFrame({ type: "VECTOR_GEN_PROGRESS", phase: "ALL", current: -1, total: 0 }); postToFrame({ type: "VECTOR_GEN_PROGRESS", phase: "ALL", current: -1, total: 0 });
break; break;
@@ -1338,7 +1347,12 @@ async function handleManualGenerate(mesId, config) {
const store = getSummaryStore(); const store = getSummaryStore();
postToFrame({ type: "SUMMARY_FULL_DATA", payload: buildFramePayload(store) }); postToFrame({ type: "SUMMARY_FULL_DATA", payload: buildFramePayload(store) });
invalidateLexicalIndex(); // Incrementally add new events to the lexical index
if (newEventIds?.length) {
const allEvents = store?.json?.events || [];
const idSet = new Set(newEventIds);
addEventDocuments(allEvents.filter(e => idSet.has(e.id)));
}
applyHideStateDebounced(); applyHideStateDebounced();
updateFrameStatsAfterSummary(endMesId, store.json || {}); updateFrameStatsAfterSummary(endMesId, store.json || {});
@@ -1381,6 +1395,10 @@ async function handleChatChanged() {
// 实体词典注入 + 索引预热 // 实体词典注入 + 索引预热
refreshEntityLexiconAndWarmup(); refreshEntityLexiconAndWarmup();
// Full lexical index rebuild on chat change
invalidateLexicalIndex();
warmupIndex();
// Embedding 连接预热(保持 TCP keep-alive减少首次召回超时 // Embedding 连接预热(保持 TCP keep-alive减少首次召回超时
warmupEmbeddingConnection(); warmupEmbeddingConnection();
@@ -1421,7 +1439,7 @@ async function handleMessageSwiped() {
await deleteStateVectorsFromFloor(chatId, lastFloor); await deleteStateVectorsFromFloor(chatId, lastFloor);
} }
invalidateLexicalIndex(); removeDocumentsByFloor(lastFloor);
initButtonsForAll(); initButtonsForAll();
applyHideStateDebounced(); applyHideStateDebounced();
@@ -1437,22 +1455,28 @@ async function handleMessageReceived() {
initButtonsForAll(); initButtonsForAll();
// 向量全量生成中时跳过 L1 sync避免竞争写入 // Skip L1 sync while full vector generation is running
if (guard.isRunning('vector')) return; if (guard.isRunning('vector')) return;
await syncOnMessageReceived(chatId, lastFloor, message, vectorConfig, () => { const syncResult = await syncOnMessageReceived(chatId, lastFloor, message, vectorConfig, () => {
sendAnchorStatsToFrame(); sendAnchorStatsToFrame();
sendVectorStatsToFrame(); sendVectorStatsToFrame();
}); });
// Incrementally update lexical index with built chunks (avoid re-read)
if (syncResult?.chunks?.length) {
addDocumentsForFloor(lastFloor, syncResult.chunks);
}
await maybeAutoBuildChunks(); await maybeAutoBuildChunks();
applyHideStateDebounced(); applyHideStateDebounced();
setTimeout(() => maybeAutoRunSummary("after_ai"), 1000); setTimeout(() => maybeAutoRunSummary("after_ai"), 1000);
// 新消息后刷新实体词典(可能有新角色) // Refresh entity lexicon after new message (new roles may appear)
refreshEntityLexiconAndWarmup(); refreshEntityLexiconAndWarmup();
// 自动补提取缺失的 L0延迟执行避免与当前楼提取竞争 // Auto backfill missing L0 (delay to avoid contention with current floor)
setTimeout(() => maybeAutoExtractL0(), 2000); setTimeout(() => maybeAutoExtractL0(), 2000);
} }
@@ -1578,19 +1602,19 @@ async function handleGenerationStarted(type, _params, isDryRun) {
// ═══════════════════════════════════════════════════════════════════════════ // ═══════════════════════════════════════════════════════════════════════════
const boundHandlers = { const boundHandlers = {
chatChanged: () => setTimeout(handleChatChanged, 80), chatChanged: () => setTimeout(handleChatChanged, 80),
messageDeleted: () => setTimeout(handleMessageDeleted, 50), messageDeleted: () => setTimeout(handleMessageDeleted, 50),
messageReceived: () => setTimeout(handleMessageReceived, 150), messageReceived: () => setTimeout(handleMessageReceived, 150),
messageSent: () => setTimeout(handleMessageSent, 150), messageSent: () => setTimeout(handleMessageSent, 150),
messageSentRecall: handleMessageSentForRecall, messageSentRecall: handleMessageSentForRecall,
messageSwiped: () => setTimeout(handleMessageSwiped, 100), messageSwiped: () => setTimeout(handleMessageSwiped, 100),
messageUpdated: () => setTimeout(handleMessageUpdated, 100), messageUpdated: () => setTimeout(handleMessageUpdated, 100),
messageEdited: () => setTimeout(handleMessageUpdated, 100), messageEdited: () => setTimeout(handleMessageUpdated, 100),
userRendered: (data) => setTimeout(() => handleMessageRendered(data), 50), userRendered: (data) => setTimeout(() => handleMessageRendered(data), 50),
charRendered: (data) => setTimeout(() => handleMessageRendered(data), 50), charRendered: (data) => setTimeout(() => handleMessageRendered(data), 50),
genStarted: handleGenerationStarted, genStarted: handleGenerationStarted,
genStopped: clearExtensionPrompt, genStopped: clearExtensionPrompt,
genEnded: clearExtensionPrompt, genEnded: clearExtensionPrompt,
}; };
function registerEvents() { function registerEvents() {
@@ -1686,4 +1710,4 @@ jQuery(() => {
initStateIntegration(); initStateIntegration();
maybePreloadTokenizer(); maybePreloadTokenizer();
}); });