Fix vector build and lexical index updates
This commit is contained in:
@@ -1,4 +1,4 @@
|
|||||||
// ═══════════════════════════════════════════════════════════════════════════
|
// ═══════════════════════════════════════════════════════════════════════════
|
||||||
// Story Summary - 主入口
|
// Story Summary - 主入口
|
||||||
//
|
//
|
||||||
// 稳定目标:
|
// 稳定目标:
|
||||||
@@ -91,7 +91,7 @@ import {
|
|||||||
// vector io
|
// vector io
|
||||||
import { exportVectors, importVectors } from "./vector/storage/vector-io.js";
|
import { exportVectors, importVectors } from "./vector/storage/vector-io.js";
|
||||||
|
|
||||||
import { invalidateLexicalIndex, warmupIndex } from "./vector/retrieval/lexical-index.js";
|
import { invalidateLexicalIndex, warmupIndex, addDocumentsForFloor, removeDocumentsByFloor, addEventDocuments } from "./vector/retrieval/lexical-index.js";
|
||||||
|
|
||||||
// ═══════════════════════════════════════════════════════════════════════════
|
// ═══════════════════════════════════════════════════════════════════════════
|
||||||
// 常量
|
// 常量
|
||||||
@@ -351,9 +351,12 @@ async function handleAnchorGenerate() {
|
|||||||
});
|
});
|
||||||
|
|
||||||
postToFrame({ type: "ANCHOR_GEN_PROGRESS", current: 0, total: 1, message: "向量化 L1..." });
|
postToFrame({ type: "ANCHOR_GEN_PROGRESS", current: 0, total: 1, message: "向量化 L1..." });
|
||||||
await buildIncrementalChunks({ vectorConfig: vectorCfg });
|
const chunkResult = await buildIncrementalChunks({ vectorConfig: vectorCfg });
|
||||||
|
|
||||||
invalidateLexicalIndex();
|
// L1 rebuild only if new chunks were added (usually 0 in normal chat)
|
||||||
|
if (chunkResult.built > 0) {
|
||||||
|
invalidateLexicalIndex();
|
||||||
|
}
|
||||||
|
|
||||||
await sendAnchorStatsToFrame();
|
await sendAnchorStatsToFrame();
|
||||||
await sendVectorStatsToFrame();
|
await sendVectorStatsToFrame();
|
||||||
@@ -589,8 +592,6 @@ function refreshEntityLexiconAndWarmup() {
|
|||||||
injectEntities(lexicon, displayMap);
|
injectEntities(lexicon, displayMap);
|
||||||
|
|
||||||
// 异步预建词法索引(不阻塞)
|
// 异步预建词法索引(不阻塞)
|
||||||
invalidateLexicalIndex();
|
|
||||||
warmupIndex();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// ═══════════════════════════════════════════════════════════════════════════
|
// ═══════════════════════════════════════════════════════════════════════════
|
||||||
@@ -615,9 +616,12 @@ async function maybeAutoExtractL0() {
|
|||||||
await incrementalExtractAtoms(chatId, chat, null, { maxFloors: 20 });
|
await incrementalExtractAtoms(chatId, chat, null, { maxFloors: 20 });
|
||||||
|
|
||||||
// 为新提取的 L0 楼层构建 L1 chunks
|
// 为新提取的 L0 楼层构建 L1 chunks
|
||||||
await buildIncrementalChunks({ vectorConfig: vectorCfg });
|
const chunkResult = await buildIncrementalChunks({ vectorConfig: vectorCfg });
|
||||||
|
|
||||||
invalidateLexicalIndex();
|
// L1 rebuild only if new chunks were added
|
||||||
|
if (chunkResult.built > 0) {
|
||||||
|
invalidateLexicalIndex();
|
||||||
|
}
|
||||||
|
|
||||||
await sendAnchorStatsToFrame();
|
await sendAnchorStatsToFrame();
|
||||||
await sendVectorStatsToFrame();
|
await sendVectorStatsToFrame();
|
||||||
@@ -637,7 +641,7 @@ async function maybeAutoExtractL0() {
|
|||||||
function warmupEmbeddingConnection() {
|
function warmupEmbeddingConnection() {
|
||||||
const vectorCfg = getVectorConfig();
|
const vectorCfg = getVectorConfig();
|
||||||
if (!vectorCfg?.enabled) return;
|
if (!vectorCfg?.enabled) return;
|
||||||
embed(['.'], vectorCfg, { timeout: 5000 }).catch(() => {});
|
embed(['.'], vectorCfg, { timeout: 5000 }).catch(() => { });
|
||||||
}
|
}
|
||||||
|
|
||||||
async function autoVectorizeNewEvents(newEventIds) {
|
async function autoVectorizeNewEvents(newEventIds) {
|
||||||
@@ -1043,7 +1047,12 @@ async function autoRunSummaryWithRetry(targetMesId, configForRun) {
|
|||||||
const store = getSummaryStore();
|
const store = getSummaryStore();
|
||||||
postToFrame({ type: "SUMMARY_FULL_DATA", payload: buildFramePayload(store) });
|
postToFrame({ type: "SUMMARY_FULL_DATA", payload: buildFramePayload(store) });
|
||||||
|
|
||||||
invalidateLexicalIndex();
|
// Incrementally add new events to the lexical index
|
||||||
|
if (newEventIds?.length) {
|
||||||
|
const allEvents = store?.json?.events || [];
|
||||||
|
const idSet = new Set(newEventIds);
|
||||||
|
addEventDocuments(allEvents.filter(e => idSet.has(e.id)));
|
||||||
|
}
|
||||||
|
|
||||||
applyHideStateDebounced();
|
applyHideStateDebounced();
|
||||||
updateFrameStatsAfterSummary(endMesId, store.json || {});
|
updateFrameStatsAfterSummary(endMesId, store.json || {});
|
||||||
@@ -1150,7 +1159,7 @@ function handleFrameMessage(event) {
|
|||||||
case "VECTOR_CANCEL_GENERATE":
|
case "VECTOR_CANCEL_GENERATE":
|
||||||
vectorCancelled = true;
|
vectorCancelled = true;
|
||||||
cancelL0Extraction();
|
cancelL0Extraction();
|
||||||
try { vectorAbortController?.abort?.(); } catch {}
|
try { vectorAbortController?.abort?.(); } catch { }
|
||||||
postToFrame({ type: "VECTOR_GEN_PROGRESS", phase: "ALL", current: -1, total: 0 });
|
postToFrame({ type: "VECTOR_GEN_PROGRESS", phase: "ALL", current: -1, total: 0 });
|
||||||
break;
|
break;
|
||||||
|
|
||||||
@@ -1338,7 +1347,12 @@ async function handleManualGenerate(mesId, config) {
|
|||||||
const store = getSummaryStore();
|
const store = getSummaryStore();
|
||||||
postToFrame({ type: "SUMMARY_FULL_DATA", payload: buildFramePayload(store) });
|
postToFrame({ type: "SUMMARY_FULL_DATA", payload: buildFramePayload(store) });
|
||||||
|
|
||||||
invalidateLexicalIndex();
|
// Incrementally add new events to the lexical index
|
||||||
|
if (newEventIds?.length) {
|
||||||
|
const allEvents = store?.json?.events || [];
|
||||||
|
const idSet = new Set(newEventIds);
|
||||||
|
addEventDocuments(allEvents.filter(e => idSet.has(e.id)));
|
||||||
|
}
|
||||||
|
|
||||||
applyHideStateDebounced();
|
applyHideStateDebounced();
|
||||||
updateFrameStatsAfterSummary(endMesId, store.json || {});
|
updateFrameStatsAfterSummary(endMesId, store.json || {});
|
||||||
@@ -1381,6 +1395,10 @@ async function handleChatChanged() {
|
|||||||
// 实体词典注入 + 索引预热
|
// 实体词典注入 + 索引预热
|
||||||
refreshEntityLexiconAndWarmup();
|
refreshEntityLexiconAndWarmup();
|
||||||
|
|
||||||
|
// Full lexical index rebuild on chat change
|
||||||
|
invalidateLexicalIndex();
|
||||||
|
warmupIndex();
|
||||||
|
|
||||||
// Embedding 连接预热(保持 TCP keep-alive,减少首次召回超时)
|
// Embedding 连接预热(保持 TCP keep-alive,减少首次召回超时)
|
||||||
warmupEmbeddingConnection();
|
warmupEmbeddingConnection();
|
||||||
|
|
||||||
@@ -1421,7 +1439,7 @@ async function handleMessageSwiped() {
|
|||||||
await deleteStateVectorsFromFloor(chatId, lastFloor);
|
await deleteStateVectorsFromFloor(chatId, lastFloor);
|
||||||
}
|
}
|
||||||
|
|
||||||
invalidateLexicalIndex();
|
removeDocumentsByFloor(lastFloor);
|
||||||
|
|
||||||
initButtonsForAll();
|
initButtonsForAll();
|
||||||
applyHideStateDebounced();
|
applyHideStateDebounced();
|
||||||
@@ -1437,22 +1455,28 @@ async function handleMessageReceived() {
|
|||||||
|
|
||||||
initButtonsForAll();
|
initButtonsForAll();
|
||||||
|
|
||||||
// 向量全量生成中时跳过 L1 sync(避免竞争写入)
|
// Skip L1 sync while full vector generation is running
|
||||||
if (guard.isRunning('vector')) return;
|
if (guard.isRunning('vector')) return;
|
||||||
|
|
||||||
await syncOnMessageReceived(chatId, lastFloor, message, vectorConfig, () => {
|
const syncResult = await syncOnMessageReceived(chatId, lastFloor, message, vectorConfig, () => {
|
||||||
sendAnchorStatsToFrame();
|
sendAnchorStatsToFrame();
|
||||||
sendVectorStatsToFrame();
|
sendVectorStatsToFrame();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// Incrementally update lexical index with built chunks (avoid re-read)
|
||||||
|
if (syncResult?.chunks?.length) {
|
||||||
|
addDocumentsForFloor(lastFloor, syncResult.chunks);
|
||||||
|
}
|
||||||
|
|
||||||
await maybeAutoBuildChunks();
|
await maybeAutoBuildChunks();
|
||||||
|
|
||||||
applyHideStateDebounced();
|
applyHideStateDebounced();
|
||||||
setTimeout(() => maybeAutoRunSummary("after_ai"), 1000);
|
setTimeout(() => maybeAutoRunSummary("after_ai"), 1000);
|
||||||
|
|
||||||
// 新消息后刷新实体词典(可能有新角色)
|
// Refresh entity lexicon after new message (new roles may appear)
|
||||||
refreshEntityLexiconAndWarmup();
|
refreshEntityLexiconAndWarmup();
|
||||||
|
|
||||||
// 自动补提取缺失的 L0(延迟执行,避免与当前楼提取竞争)
|
// Auto backfill missing L0 (delay to avoid contention with current floor)
|
||||||
setTimeout(() => maybeAutoExtractL0(), 2000);
|
setTimeout(() => maybeAutoExtractL0(), 2000);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1578,19 +1602,19 @@ async function handleGenerationStarted(type, _params, isDryRun) {
|
|||||||
// ═══════════════════════════════════════════════════════════════════════════
|
// ═══════════════════════════════════════════════════════════════════════════
|
||||||
|
|
||||||
const boundHandlers = {
|
const boundHandlers = {
|
||||||
chatChanged: () => setTimeout(handleChatChanged, 80),
|
chatChanged: () => setTimeout(handleChatChanged, 80),
|
||||||
messageDeleted: () => setTimeout(handleMessageDeleted, 50),
|
messageDeleted: () => setTimeout(handleMessageDeleted, 50),
|
||||||
messageReceived: () => setTimeout(handleMessageReceived, 150),
|
messageReceived: () => setTimeout(handleMessageReceived, 150),
|
||||||
messageSent: () => setTimeout(handleMessageSent, 150),
|
messageSent: () => setTimeout(handleMessageSent, 150),
|
||||||
messageSentRecall: handleMessageSentForRecall,
|
messageSentRecall: handleMessageSentForRecall,
|
||||||
messageSwiped: () => setTimeout(handleMessageSwiped, 100),
|
messageSwiped: () => setTimeout(handleMessageSwiped, 100),
|
||||||
messageUpdated: () => setTimeout(handleMessageUpdated, 100),
|
messageUpdated: () => setTimeout(handleMessageUpdated, 100),
|
||||||
messageEdited: () => setTimeout(handleMessageUpdated, 100),
|
messageEdited: () => setTimeout(handleMessageUpdated, 100),
|
||||||
userRendered: (data) => setTimeout(() => handleMessageRendered(data), 50),
|
userRendered: (data) => setTimeout(() => handleMessageRendered(data), 50),
|
||||||
charRendered: (data) => setTimeout(() => handleMessageRendered(data), 50),
|
charRendered: (data) => setTimeout(() => handleMessageRendered(data), 50),
|
||||||
genStarted: handleGenerationStarted,
|
genStarted: handleGenerationStarted,
|
||||||
genStopped: clearExtensionPrompt,
|
genStopped: clearExtensionPrompt,
|
||||||
genEnded: clearExtensionPrompt,
|
genEnded: clearExtensionPrompt,
|
||||||
};
|
};
|
||||||
|
|
||||||
function registerEvents() {
|
function registerEvents() {
|
||||||
|
|||||||
Reference in New Issue
Block a user