2.0变量 , 向量总结正式推送
This commit is contained in:
RT15548
261
modules/story-summary/vector/storage/chunk-store.js
Normal file
261
modules/story-summary/vector/storage/chunk-store.js
Normal file
@@ -0,0 +1,261 @@
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
// Story Summary - Chunk Store (L1/L2 storage)
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
|
||||
import {
|
||||
metaTable,
|
||||
chunksTable,
|
||||
chunkVectorsTable,
|
||||
eventVectorsTable,
|
||||
CHUNK_MAX_TOKENS,
|
||||
} from '../../data/db.js';
|
||||
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
// 工具函数
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
|
||||
export function float32ToBuffer(arr) {
|
||||
return arr.buffer.slice(arr.byteOffset, arr.byteOffset + arr.byteLength);
|
||||
}
|
||||
|
||||
export function bufferToFloat32(buffer) {
|
||||
return new Float32Array(buffer);
|
||||
}
|
||||
|
||||
export function makeChunkId(floor, chunkIdx) {
|
||||
return `c-${floor}-${chunkIdx}`;
|
||||
}
|
||||
|
||||
export function hashText(text) {
|
||||
let hash = 0;
|
||||
for (let i = 0; i < text.length; i++) {
|
||||
hash = ((hash << 5) - hash + text.charCodeAt(i)) | 0;
|
||||
}
|
||||
return hash.toString(36);
|
||||
}
|
||||
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
// Meta 表操作
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
|
||||
export async function getMeta(chatId) {
|
||||
let meta = await metaTable.get(chatId);
|
||||
if (!meta) {
|
||||
meta = {
|
||||
chatId,
|
||||
fingerprint: null,
|
||||
lastChunkFloor: -1,
|
||||
updatedAt: Date.now(),
|
||||
};
|
||||
await metaTable.put(meta);
|
||||
}
|
||||
return meta;
|
||||
}
|
||||
|
||||
export async function updateMeta(chatId, updates) {
|
||||
await metaTable.update(chatId, {
|
||||
...updates,
|
||||
updatedAt: Date.now(),
|
||||
});
|
||||
}
|
||||
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
// Chunks 表操作
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
|
||||
export async function saveChunks(chatId, chunks) {
|
||||
const records = chunks.map(chunk => ({
|
||||
chatId,
|
||||
chunkId: chunk.chunkId,
|
||||
floor: chunk.floor,
|
||||
chunkIdx: chunk.chunkIdx,
|
||||
speaker: chunk.speaker,
|
||||
isUser: chunk.isUser,
|
||||
text: chunk.text,
|
||||
textHash: chunk.textHash,
|
||||
createdAt: Date.now(),
|
||||
}));
|
||||
await chunksTable.bulkPut(records);
|
||||
}
|
||||
|
||||
export async function getAllChunks(chatId) {
|
||||
return await chunksTable.where('chatId').equals(chatId).toArray();
|
||||
}
|
||||
|
||||
export async function getChunksByFloors(chatId, floors) {
|
||||
const chunks = await chunksTable
|
||||
.where('[chatId+floor]')
|
||||
.anyOf(floors.map(f => [chatId, f]))
|
||||
.toArray();
|
||||
return chunks;
|
||||
}
|
||||
|
||||
/**
|
||||
* 删除指定楼层及之后的所有 chunk 和向量
|
||||
*/
|
||||
export async function deleteChunksFromFloor(chatId, fromFloor) {
|
||||
const chunks = await chunksTable
|
||||
.where('chatId')
|
||||
.equals(chatId)
|
||||
.filter(c => c.floor >= fromFloor)
|
||||
.toArray();
|
||||
|
||||
const chunkIds = chunks.map(c => c.chunkId);
|
||||
|
||||
await chunksTable
|
||||
.where('chatId')
|
||||
.equals(chatId)
|
||||
.filter(c => c.floor >= fromFloor)
|
||||
.delete();
|
||||
|
||||
for (const chunkId of chunkIds) {
|
||||
await chunkVectorsTable.delete([chatId, chunkId]);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 删除指定楼层的 chunk 和向量
|
||||
*/
|
||||
export async function deleteChunksAtFloor(chatId, floor) {
|
||||
const chunks = await chunksTable
|
||||
.where('[chatId+floor]')
|
||||
.equals([chatId, floor])
|
||||
.toArray();
|
||||
|
||||
const chunkIds = chunks.map(c => c.chunkId);
|
||||
|
||||
await chunksTable.where('[chatId+floor]').equals([chatId, floor]).delete();
|
||||
|
||||
for (const chunkId of chunkIds) {
|
||||
await chunkVectorsTable.delete([chatId, chunkId]);
|
||||
}
|
||||
}
|
||||
|
||||
export async function clearAllChunks(chatId) {
|
||||
await chunksTable.where('chatId').equals(chatId).delete();
|
||||
await chunkVectorsTable.where('chatId').equals(chatId).delete();
|
||||
}
|
||||
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
// ChunkVectors 表操作
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
|
||||
export async function saveChunkVectors(chatId, items, fingerprint) {
|
||||
const records = items.map(item => ({
|
||||
chatId,
|
||||
chunkId: item.chunkId,
|
||||
vector: float32ToBuffer(new Float32Array(item.vector)),
|
||||
dims: item.vector.length,
|
||||
fingerprint,
|
||||
}));
|
||||
await chunkVectorsTable.bulkPut(records);
|
||||
}
|
||||
|
||||
export async function getAllChunkVectors(chatId) {
|
||||
const records = await chunkVectorsTable.where('chatId').equals(chatId).toArray();
|
||||
return records.map(r => ({
|
||||
...r,
|
||||
vector: bufferToFloat32(r.vector),
|
||||
}));
|
||||
}
|
||||
|
||||
export async function getChunkVectorsByIds(chatId, chunkIds) {
|
||||
if (!chatId || !chunkIds?.length) return [];
|
||||
|
||||
const records = await chunkVectorsTable
|
||||
.where('[chatId+chunkId]')
|
||||
.anyOf(chunkIds.map(id => [chatId, id]))
|
||||
.toArray();
|
||||
|
||||
return records.map(r => ({
|
||||
chunkId: r.chunkId,
|
||||
vector: bufferToFloat32(r.vector),
|
||||
}));
|
||||
}
|
||||
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
// EventVectors 表操作
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
|
||||
export async function saveEventVectors(chatId, items, fingerprint) {
|
||||
const records = items.map(item => ({
|
||||
chatId,
|
||||
eventId: item.eventId,
|
||||
vector: float32ToBuffer(new Float32Array(item.vector)),
|
||||
dims: item.vector.length,
|
||||
fingerprint,
|
||||
}));
|
||||
await eventVectorsTable.bulkPut(records);
|
||||
}
|
||||
|
||||
export async function getAllEventVectors(chatId) {
|
||||
const records = await eventVectorsTable.where('chatId').equals(chatId).toArray();
|
||||
return records.map(r => ({
|
||||
...r,
|
||||
vector: bufferToFloat32(r.vector),
|
||||
}));
|
||||
}
|
||||
|
||||
export async function clearEventVectors(chatId) {
|
||||
await eventVectorsTable.where('chatId').equals(chatId).delete();
|
||||
}
|
||||
|
||||
/**
|
||||
* 按 ID 列表删除 event 向量
|
||||
*/
|
||||
export async function deleteEventVectorsByIds(chatId, eventIds) {
|
||||
for (const eventId of eventIds) {
|
||||
await eventVectorsTable.delete([chatId, eventId]);
|
||||
}
|
||||
}
|
||||
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
// 统计与工具
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
|
||||
export async function getStorageStats(chatId) {
|
||||
const [meta, chunkCount, chunkVectorCount, eventCount] = await Promise.all([
|
||||
getMeta(chatId),
|
||||
chunksTable.where('chatId').equals(chatId).count(),
|
||||
chunkVectorsTable.where('chatId').equals(chatId).count(),
|
||||
eventVectorsTable.where('chatId').equals(chatId).count(),
|
||||
]);
|
||||
|
||||
return {
|
||||
fingerprint: meta.fingerprint,
|
||||
lastChunkFloor: meta.lastChunkFloor,
|
||||
chunks: chunkCount,
|
||||
chunkVectors: chunkVectorCount,
|
||||
eventVectors: eventCount,
|
||||
};
|
||||
}
|
||||
|
||||
export async function clearChatData(chatId) {
|
||||
await Promise.all([
|
||||
metaTable.delete(chatId),
|
||||
chunksTable.where('chatId').equals(chatId).delete(),
|
||||
chunkVectorsTable.where('chatId').equals(chatId).delete(),
|
||||
eventVectorsTable.where('chatId').equals(chatId).delete(),
|
||||
]);
|
||||
}
|
||||
|
||||
export async function ensureFingerprintMatch(chatId, newFingerprint) {
|
||||
const meta = await getMeta(chatId);
|
||||
if (meta.fingerprint && meta.fingerprint !== newFingerprint) {
|
||||
await Promise.all([
|
||||
chunkVectorsTable.where('chatId').equals(chatId).delete(),
|
||||
eventVectorsTable.where('chatId').equals(chatId).delete(),
|
||||
]);
|
||||
await updateMeta(chatId, {
|
||||
fingerprint: newFingerprint,
|
||||
lastChunkFloor: -1,
|
||||
});
|
||||
return false;
|
||||
}
|
||||
if (!meta.fingerprint) {
|
||||
await updateMeta(chatId, { fingerprint: newFingerprint });
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
export { CHUNK_MAX_TOKENS };
|
||||
266
modules/story-summary/vector/storage/state-store.js
Normal file
266
modules/story-summary/vector/storage/state-store.js
Normal file
@@ -0,0 +1,266 @@
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
// Story Summary - State Store (L0)
|
||||
// StateAtom 存 chat_metadata（持久化）
|
||||
// StateVector 存 IndexedDB（可重建）
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
|
||||
import { saveMetadataDebounced } from '../../../../../../../extensions.js';
|
||||
import { chat_metadata } from '../../../../../../../../script.js';
|
||||
import { stateVectorsTable } from '../../data/db.js';
|
||||
import { EXT_ID } from '../../../../core/constants.js';
|
||||
import { xbLog } from '../../../../core/debug-core.js';
|
||||
|
||||
const MODULE_ID = 'state-store';
|
||||
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
// 工具函数
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
|
||||
export function float32ToBuffer(arr) {
|
||||
return arr.buffer.slice(arr.byteOffset, arr.byteOffset + arr.byteLength);
|
||||
}
|
||||
|
||||
export function bufferToFloat32(buffer) {
|
||||
return new Float32Array(buffer);
|
||||
}
|
||||
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
// StateAtom 操作（chat_metadata）
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
|
||||
function ensureStateAtomsArray() {
|
||||
chat_metadata.extensions ||= {};
|
||||
chat_metadata.extensions[EXT_ID] ||= {};
|
||||
chat_metadata.extensions[EXT_ID].stateAtoms ||= [];
|
||||
return chat_metadata.extensions[EXT_ID].stateAtoms;
|
||||
}
|
||||
|
||||
// L0Index: per-floor status (ok | empty | fail)
|
||||
function ensureL0Index() {
|
||||
chat_metadata.extensions ||= {};
|
||||
chat_metadata.extensions[EXT_ID] ||= {};
|
||||
chat_metadata.extensions[EXT_ID].l0Index ||= { version: 1, byFloor: {} };
|
||||
chat_metadata.extensions[EXT_ID].l0Index.byFloor ||= {};
|
||||
return chat_metadata.extensions[EXT_ID].l0Index;
|
||||
}
|
||||
|
||||
export function getL0Index() {
|
||||
return ensureL0Index();
|
||||
}
|
||||
|
||||
export function getL0FloorStatus(floor) {
|
||||
const idx = ensureL0Index();
|
||||
return idx.byFloor?.[String(floor)] || null;
|
||||
}
|
||||
|
||||
export function setL0FloorStatus(floor, record) {
|
||||
const idx = ensureL0Index();
|
||||
idx.byFloor[String(floor)] = {
|
||||
...record,
|
||||
floor,
|
||||
updatedAt: Date.now(),
|
||||
};
|
||||
saveMetadataDebounced();
|
||||
}
|
||||
|
||||
export function clearL0Index() {
|
||||
const idx = ensureL0Index();
|
||||
idx.byFloor = {};
|
||||
saveMetadataDebounced();
|
||||
}
|
||||
|
||||
export function deleteL0IndexFromFloor(fromFloor) {
|
||||
const idx = ensureL0Index();
|
||||
const keys = Object.keys(idx.byFloor || {});
|
||||
let deleted = 0;
|
||||
for (const k of keys) {
|
||||
const f = Number(k);
|
||||
if (Number.isFinite(f) && f >= fromFloor) {
|
||||
delete idx.byFloor[k];
|
||||
deleted++;
|
||||
}
|
||||
}
|
||||
if (deleted > 0) {
|
||||
saveMetadataDebounced();
|
||||
xbLog.info(MODULE_ID, `删除 ${deleted} 条 L0Index (floor >= ${fromFloor})`);
|
||||
}
|
||||
return deleted;
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取当前聊天的所有 StateAtoms
|
||||
*/
|
||||
export function getStateAtoms() {
|
||||
return ensureStateAtomsArray();
|
||||
}
|
||||
|
||||
/**
|
||||
* 保存新的 StateAtoms（追加，去重）
|
||||
*/
|
||||
export function saveStateAtoms(atoms) {
|
||||
if (!atoms?.length) return;
|
||||
|
||||
const arr = ensureStateAtomsArray();
|
||||
const existing = new Set(arr.map(a => a.atomId));
|
||||
|
||||
let added = 0;
|
||||
for (const atom of atoms) {
|
||||
// 有效性检查
|
||||
if (!atom?.atomId || typeof atom.floor !== 'number' || atom.floor < 0 || !atom.semantic) {
|
||||
xbLog.warn(MODULE_ID, `跳过无效 atom: ${atom?.atomId}`);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!existing.has(atom.atomId)) {
|
||||
arr.push(atom);
|
||||
existing.add(atom.atomId);
|
||||
added++;
|
||||
}
|
||||
}
|
||||
|
||||
if (added > 0) {
|
||||
saveMetadataDebounced();
|
||||
xbLog.info(MODULE_ID, `存储 ${added} 个 StateAtom`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 删除指定楼层及之后的 StateAtoms
|
||||
*/
|
||||
export function deleteStateAtomsFromFloor(floor) {
|
||||
const arr = ensureStateAtomsArray();
|
||||
const before = arr.length;
|
||||
|
||||
const filtered = arr.filter(a => a.floor < floor);
|
||||
chat_metadata.extensions[EXT_ID].stateAtoms = filtered;
|
||||
|
||||
const deleted = before - filtered.length;
|
||||
if (deleted > 0) {
|
||||
saveMetadataDebounced();
|
||||
xbLog.info(MODULE_ID, `删除 ${deleted} 个 StateAtom (floor >= ${floor})`);
|
||||
}
|
||||
|
||||
return deleted;
|
||||
}
|
||||
|
||||
/**
|
||||
* 清空所有 StateAtoms
|
||||
*/
|
||||
export function clearStateAtoms() {
|
||||
const arr = ensureStateAtomsArray();
|
||||
const count = arr.length;
|
||||
|
||||
chat_metadata.extensions[EXT_ID].stateAtoms = [];
|
||||
|
||||
if (count > 0) {
|
||||
saveMetadataDebounced();
|
||||
xbLog.info(MODULE_ID, `清空 ${count} 个 StateAtom`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取 StateAtoms 数量
|
||||
*/
|
||||
export function getStateAtomsCount() {
|
||||
return ensureStateAtomsArray().length;
|
||||
}
|
||||
|
||||
/**
|
||||
* Return floors that already have extracted atoms.
|
||||
*/
|
||||
export function getExtractedFloors() {
|
||||
const floors = new Set();
|
||||
const arr = ensureStateAtomsArray();
|
||||
for (const atom of arr) {
|
||||
if (typeof atom?.floor === 'number' && atom.floor >= 0) {
|
||||
floors.add(atom.floor);
|
||||
}
|
||||
}
|
||||
return floors;
|
||||
}
|
||||
|
||||
/**
|
||||
* Replace all stored StateAtoms.
|
||||
*/
|
||||
export function replaceStateAtoms(atoms) {
|
||||
const next = Array.isArray(atoms) ? atoms : [];
|
||||
chat_metadata.extensions[EXT_ID].stateAtoms = next;
|
||||
saveMetadataDebounced();
|
||||
xbLog.info(MODULE_ID, `替换 StateAtoms: ${next.length} 条`);
|
||||
}
|
||||
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
// StateVector 操作（IndexedDB）
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
|
||||
/**
|
||||
* 保存 StateVectors
|
||||
*/
|
||||
export async function saveStateVectors(chatId, items, fingerprint) {
|
||||
if (!chatId || !items?.length) return;
|
||||
|
||||
const records = items.map(item => ({
|
||||
chatId,
|
||||
atomId: item.atomId,
|
||||
floor: item.floor,
|
||||
vector: float32ToBuffer(new Float32Array(item.vector)),
|
||||
dims: item.vector.length,
|
||||
rVector: item.rVector?.length ? float32ToBuffer(new Float32Array(item.rVector)) : null,
|
||||
rDims: item.rVector?.length ? item.rVector.length : 0,
|
||||
fingerprint,
|
||||
}));
|
||||
|
||||
await stateVectorsTable.bulkPut(records);
|
||||
xbLog.info(MODULE_ID, `存储 ${records.length} 个 StateVector`);
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取所有 StateVectors
|
||||
*/
|
||||
export async function getAllStateVectors(chatId) {
|
||||
if (!chatId) return [];
|
||||
|
||||
const records = await stateVectorsTable.where('chatId').equals(chatId).toArray();
|
||||
return records.map(r => ({
|
||||
...r,
|
||||
vector: bufferToFloat32(r.vector),
|
||||
rVector: r.rVector ? bufferToFloat32(r.rVector) : null,
|
||||
}));
|
||||
}
|
||||
|
||||
/**
|
||||
* 删除指定楼层及之后的 StateVectors
|
||||
*/
|
||||
export async function deleteStateVectorsFromFloor(chatId, floor) {
|
||||
if (!chatId) return;
|
||||
|
||||
const deleted = await stateVectorsTable
|
||||
.where('chatId')
|
||||
.equals(chatId)
|
||||
.filter(v => v.floor >= floor)
|
||||
.delete();
|
||||
|
||||
if (deleted > 0) {
|
||||
xbLog.info(MODULE_ID, `删除 ${deleted} 个 StateVector (floor >= ${floor})`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 清空所有 StateVectors
|
||||
*/
|
||||
export async function clearStateVectors(chatId) {
|
||||
if (!chatId) return;
|
||||
|
||||
const deleted = await stateVectorsTable.where('chatId').equals(chatId).delete();
|
||||
if (deleted > 0) {
|
||||
xbLog.info(MODULE_ID, `清空 ${deleted} 个 StateVector`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取 StateVectors 数量
|
||||
*/
|
||||
export async function getStateVectorsCount(chatId) {
|
||||
if (!chatId) return 0;
|
||||
return await stateVectorsTable.where('chatId').equals(chatId).count();
|
||||
}
|
||||
385
modules/story-summary/vector/storage/vector-io.js
Normal file
385
modules/story-summary/vector/storage/vector-io.js
Normal file
@@ -0,0 +1,385 @@
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
// Vector Import/Export
|
||||
// 向量数据导入导出（当前 chatId 级别）
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
|
||||
import { zipSync, unzipSync, strToU8, strFromU8 } from '../../../../libs/fflate.mjs';
|
||||
import { getContext } from '../../../../../../../extensions.js';
|
||||
import { xbLog } from '../../../../core/debug-core.js';
|
||||
import {
|
||||
getMeta,
|
||||
updateMeta,
|
||||
getAllChunks,
|
||||
getAllChunkVectors,
|
||||
getAllEventVectors,
|
||||
saveChunks,
|
||||
saveChunkVectors,
|
||||
clearAllChunks,
|
||||
clearEventVectors,
|
||||
saveEventVectors,
|
||||
} from './chunk-store.js';
|
||||
import {
|
||||
getStateAtoms,
|
||||
saveStateAtoms,
|
||||
clearStateAtoms,
|
||||
getAllStateVectors,
|
||||
saveStateVectors,
|
||||
clearStateVectors,
|
||||
} from './state-store.js';
|
||||
import { getEngineFingerprint } from '../utils/embedder.js';
|
||||
import { getVectorConfig } from '../../data/config.js';
|
||||
|
||||
const MODULE_ID = 'vector-io';
|
||||
const EXPORT_VERSION = 2;
|
||||
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
// 工具函数
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
|
||||
function float32ToBytes(vectors, dims) {
|
||||
const totalFloats = vectors.length * dims;
|
||||
const buffer = new ArrayBuffer(totalFloats * 4);
|
||||
const view = new Float32Array(buffer);
|
||||
|
||||
let offset = 0;
|
||||
for (const vec of vectors) {
|
||||
for (let i = 0; i < dims; i++) {
|
||||
view[offset++] = vec[i] || 0;
|
||||
}
|
||||
}
|
||||
|
||||
return new Uint8Array(buffer);
|
||||
}
|
||||
|
||||
function bytesToFloat32(bytes, dims) {
|
||||
const view = new Float32Array(bytes.buffer, bytes.byteOffset, bytes.byteLength / 4);
|
||||
const vectors = [];
|
||||
|
||||
for (let i = 0; i < view.length; i += dims) {
|
||||
vectors.push(Array.from(view.slice(i, i + dims)));
|
||||
}
|
||||
|
||||
return vectors;
|
||||
}
|
||||
|
||||
function downloadBlob(blob, filename) {
|
||||
const url = URL.createObjectURL(blob);
|
||||
const a = document.createElement('a');
|
||||
a.href = url;
|
||||
a.download = filename;
|
||||
document.body.appendChild(a);
|
||||
a.click();
|
||||
document.body.removeChild(a);
|
||||
URL.revokeObjectURL(url);
|
||||
}
|
||||
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
// 导出
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
|
||||
export async function exportVectors(onProgress) {
|
||||
const { chatId } = getContext();
|
||||
if (!chatId) {
|
||||
throw new Error('未打开聊天');
|
||||
}
|
||||
|
||||
onProgress?.('读取数据...');
|
||||
|
||||
const meta = await getMeta(chatId);
|
||||
const chunks = await getAllChunks(chatId);
|
||||
const chunkVectors = await getAllChunkVectors(chatId);
|
||||
const eventVectors = await getAllEventVectors(chatId);
|
||||
const stateAtoms = getStateAtoms();
|
||||
const stateVectors = await getAllStateVectors(chatId);
|
||||
|
||||
if (chunkVectors.length === 0 && eventVectors.length === 0 && stateVectors.length === 0) {
|
||||
throw new Error('没有可导出的向量数据');
|
||||
}
|
||||
|
||||
// 确定维度
|
||||
const dims = chunkVectors[0]?.vector?.length
|
||||
|| eventVectors[0]?.vector?.length
|
||||
|| stateVectors[0]?.vector?.length
|
||||
|| 0;
|
||||
if (dims === 0) {
|
||||
throw new Error('无法确定向量维度');
|
||||
}
|
||||
|
||||
onProgress?.('构建索引...');
|
||||
|
||||
// 构建 chunk 索引（按 chunkId 排序保证顺序一致）
|
||||
const sortedChunks = [...chunks].sort((a, b) => a.chunkId.localeCompare(b.chunkId));
|
||||
const chunkVectorMap = new Map(chunkVectors.map(cv => [cv.chunkId, cv.vector]));
|
||||
|
||||
// chunks.jsonl
|
||||
const chunksJsonl = sortedChunks.map(c => JSON.stringify({
|
||||
chunkId: c.chunkId,
|
||||
floor: c.floor,
|
||||
chunkIdx: c.chunkIdx,
|
||||
speaker: c.speaker,
|
||||
isUser: c.isUser,
|
||||
text: c.text,
|
||||
textHash: c.textHash,
|
||||
})).join('\n');
|
||||
|
||||
// chunk_vectors.bin（按 sortedChunks 顺序）
|
||||
const chunkVectorsOrdered = sortedChunks.map(c => chunkVectorMap.get(c.chunkId) || new Array(dims).fill(0));
|
||||
|
||||
onProgress?.('压缩向量...');
|
||||
|
||||
// 构建 event 索引
|
||||
const sortedEventVectors = [...eventVectors].sort((a, b) => a.eventId.localeCompare(b.eventId));
|
||||
const eventsJsonl = sortedEventVectors.map(ev => JSON.stringify({
|
||||
eventId: ev.eventId,
|
||||
})).join('\n');
|
||||
|
||||
// event_vectors.bin
|
||||
const eventVectorsOrdered = sortedEventVectors.map(ev => ev.vector);
|
||||
|
||||
// state vectors
|
||||
const sortedStateVectors = [...stateVectors].sort((a, b) => String(a.atomId).localeCompare(String(b.atomId)));
|
||||
const stateVectorsOrdered = sortedStateVectors.map(v => v.vector);
|
||||
const rDims = sortedStateVectors.find(v => v.rVector?.length)?.rVector?.length || dims;
|
||||
const stateRVectorsOrdered = sortedStateVectors.map(v =>
|
||||
v.rVector?.length ? v.rVector : new Array(rDims).fill(0)
|
||||
);
|
||||
const stateVectorsJsonl = sortedStateVectors.map(v => JSON.stringify({
|
||||
atomId: v.atomId,
|
||||
floor: v.floor,
|
||||
hasRVector: !!(v.rVector?.length),
|
||||
rDims: v.rVector?.length || 0,
|
||||
})).join('\n');
|
||||
|
||||
// manifest
|
||||
const manifest = {
|
||||
version: EXPORT_VERSION,
|
||||
exportedAt: Date.now(),
|
||||
chatId,
|
||||
fingerprint: meta.fingerprint || '',
|
||||
dims,
|
||||
chunkCount: sortedChunks.length,
|
||||
chunkVectorCount: chunkVectors.length,
|
||||
eventCount: sortedEventVectors.length,
|
||||
stateAtomCount: stateAtoms.length,
|
||||
stateVectorCount: stateVectors.length,
|
||||
stateRVectorCount: sortedStateVectors.filter(v => v.rVector?.length).length,
|
||||
rDims,
|
||||
lastChunkFloor: meta.lastChunkFloor ?? -1,
|
||||
};
|
||||
|
||||
onProgress?.('打包文件...');
|
||||
|
||||
// 打包 zip
|
||||
const zipData = zipSync({
|
||||
'manifest.json': strToU8(JSON.stringify(manifest, null, 2)),
|
||||
'chunks.jsonl': strToU8(chunksJsonl),
|
||||
'chunk_vectors.bin': float32ToBytes(chunkVectorsOrdered, dims),
|
||||
'events.jsonl': strToU8(eventsJsonl),
|
||||
'event_vectors.bin': float32ToBytes(eventVectorsOrdered, dims),
|
||||
'state_atoms.json': strToU8(JSON.stringify(stateAtoms)),
|
||||
'state_vectors.jsonl': strToU8(stateVectorsJsonl),
|
||||
'state_vectors.bin': stateVectorsOrdered.length
|
||||
? float32ToBytes(stateVectorsOrdered, dims)
|
||||
: new Uint8Array(0),
|
||||
'state_r_vectors.bin': stateRVectorsOrdered.length
|
||||
? float32ToBytes(stateRVectorsOrdered, rDims)
|
||||
: new Uint8Array(0),
|
||||
}, { level: 1 }); // 降低压缩级别，速度优先
|
||||
|
||||
onProgress?.('下载文件...');
|
||||
|
||||
// 生成文件名
|
||||
const timestamp = new Date().toISOString().slice(0, 10).replace(/-/g, '');
|
||||
const shortChatId = chatId.slice(0, 8);
|
||||
const filename = `vectors_${shortChatId}_${timestamp}.zip`;
|
||||
|
||||
downloadBlob(new Blob([zipData]), filename);
|
||||
|
||||
const sizeMB = (zipData.byteLength / 1024 / 1024).toFixed(2);
|
||||
xbLog.info(MODULE_ID, `导出完成: ${filename} (${sizeMB}MB)`);
|
||||
|
||||
return {
|
||||
filename,
|
||||
size: zipData.byteLength,
|
||||
chunkCount: sortedChunks.length,
|
||||
eventCount: sortedEventVectors.length,
|
||||
};
|
||||
}
|
||||
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
// 导入
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
|
||||
export async function importVectors(file, onProgress) {
|
||||
const { chatId } = getContext();
|
||||
if (!chatId) {
|
||||
throw new Error('未打开聊天');
|
||||
}
|
||||
|
||||
onProgress?.('读取文件...');
|
||||
|
||||
const arrayBuffer = await file.arrayBuffer();
|
||||
const zipData = new Uint8Array(arrayBuffer);
|
||||
|
||||
onProgress?.('解压文件...');
|
||||
|
||||
let unzipped;
|
||||
try {
|
||||
unzipped = unzipSync(zipData);
|
||||
} catch (e) {
|
||||
throw new Error('文件格式错误，无法解压');
|
||||
}
|
||||
|
||||
// 读取 manifest
|
||||
if (!unzipped['manifest.json']) {
|
||||
throw new Error('缺少 manifest.json');
|
||||
}
|
||||
|
||||
const manifest = JSON.parse(strFromU8(unzipped['manifest.json']));
|
||||
|
||||
if (![1, 2].includes(manifest.version)) {
|
||||
throw new Error(`不支持的版本: ${manifest.version}`);
|
||||
}
|
||||
|
||||
onProgress?.('校验数据...');
|
||||
|
||||
// 校验 fingerprint
|
||||
const vectorCfg = getVectorConfig();
|
||||
const currentFingerprint = vectorCfg ? getEngineFingerprint(vectorCfg) : '';
|
||||
const fingerprintMismatch = manifest.fingerprint && currentFingerprint && manifest.fingerprint !== currentFingerprint;
|
||||
|
||||
// chatId 校验（警告但允许）
|
||||
const chatIdMismatch = manifest.chatId !== chatId;
|
||||
|
||||
const warnings = [];
|
||||
if (fingerprintMismatch) {
|
||||
warnings.push(`向量引擎不匹配（文件: ${manifest.fingerprint}, 当前: ${currentFingerprint}），导入后需重新生成`);
|
||||
}
|
||||
if (chatIdMismatch) {
|
||||
warnings.push(`聊天ID不匹配（文件: ${manifest.chatId}, 当前: ${chatId}）`);
|
||||
}
|
||||
|
||||
onProgress?.('解析数据...');
|
||||
|
||||
// 解析 chunks
|
||||
const chunksJsonl = unzipped['chunks.jsonl'] ? strFromU8(unzipped['chunks.jsonl']) : '';
|
||||
const chunkMetas = chunksJsonl.split('\n').filter(Boolean).map(line => JSON.parse(line));
|
||||
|
||||
// 解析 chunk vectors
|
||||
const chunkVectorsBytes = unzipped['chunk_vectors.bin'];
|
||||
const chunkVectors = chunkVectorsBytes ? bytesToFloat32(chunkVectorsBytes, manifest.dims) : [];
|
||||
|
||||
// 解析 events
|
||||
const eventsJsonl = unzipped['events.jsonl'] ? strFromU8(unzipped['events.jsonl']) : '';
|
||||
const eventMetas = eventsJsonl.split('\n').filter(Boolean).map(line => JSON.parse(line));
|
||||
|
||||
// 解析 event vectors
|
||||
const eventVectorsBytes = unzipped['event_vectors.bin'];
|
||||
const eventVectors = eventVectorsBytes ? bytesToFloat32(eventVectorsBytes, manifest.dims) : [];
|
||||
|
||||
// 解析 L0 state atoms
|
||||
const stateAtoms = unzipped['state_atoms.json']
|
||||
? JSON.parse(strFromU8(unzipped['state_atoms.json']))
|
||||
: [];
|
||||
|
||||
// 解析 L0 state vectors metas
|
||||
const stateVectorsJsonl = unzipped['state_vectors.jsonl'] ? strFromU8(unzipped['state_vectors.jsonl']) : '';
|
||||
const stateVectorMetas = stateVectorsJsonl.split('\n').filter(Boolean).map(line => JSON.parse(line));
|
||||
|
||||
// Parse L0 semantic vectors
|
||||
const stateVectorsBytes = unzipped['state_vectors.bin'];
|
||||
const stateVectors = (stateVectorsBytes && stateVectorMetas.length)
|
||||
? bytesToFloat32(stateVectorsBytes, manifest.dims)
|
||||
: [];
|
||||
// Parse optional L0 r-vectors (for diffusion r-sem edges)
|
||||
const stateRVectorsBytes = unzipped['state_r_vectors.bin'];
|
||||
const stateRVectors = (stateRVectorsBytes && stateVectorMetas.length)
|
||||
? bytesToFloat32(stateRVectorsBytes, manifest.rDims || manifest.dims)
|
||||
: [];
|
||||
const hasRVectorMeta = stateVectorMetas.some(m => typeof m.hasRVector === 'boolean');
|
||||
|
||||
// 校验数量
|
||||
if (chunkMetas.length !== chunkVectors.length) {
|
||||
throw new Error(`chunk 数量不匹配: 元数据 ${chunkMetas.length}, 向量 ${chunkVectors.length}`);
|
||||
}
|
||||
if (eventMetas.length !== eventVectors.length) {
|
||||
throw new Error(`event 数量不匹配: 元数据 ${eventMetas.length}, 向量 ${eventVectors.length}`);
|
||||
}
|
||||
if (stateVectorMetas.length !== stateVectors.length) {
|
||||
throw new Error(`state 向量数量不匹配: 元数据 ${stateVectorMetas.length}, 向量 ${stateVectors.length}`);
|
||||
}
|
||||
if (stateRVectors.length > 0 && stateVectorMetas.length !== stateRVectors.length) {
|
||||
throw new Error(`state r-vector count mismatch: meta=${stateVectorMetas.length}, vectors=${stateRVectors.length}`);
|
||||
}
|
||||
|
||||
onProgress?.('清空旧数据...');
|
||||
|
||||
// 清空当前数据
|
||||
await clearAllChunks(chatId);
|
||||
await clearEventVectors(chatId);
|
||||
await clearStateVectors(chatId);
|
||||
clearStateAtoms();
|
||||
|
||||
onProgress?.('写入数据...');
|
||||
|
||||
// 写入 chunks
|
||||
if (chunkMetas.length > 0) {
|
||||
const chunksToSave = chunkMetas.map(meta => ({
|
||||
chunkId: meta.chunkId,
|
||||
floor: meta.floor,
|
||||
chunkIdx: meta.chunkIdx,
|
||||
speaker: meta.speaker,
|
||||
isUser: meta.isUser,
|
||||
text: meta.text,
|
||||
textHash: meta.textHash,
|
||||
}));
|
||||
await saveChunks(chatId, chunksToSave);
|
||||
|
||||
// 写入 chunk vectors
|
||||
const chunkVectorItems = chunkMetas.map((meta, idx) => ({
|
||||
chunkId: meta.chunkId,
|
||||
vector: chunkVectors[idx],
|
||||
}));
|
||||
await saveChunkVectors(chatId, chunkVectorItems, manifest.fingerprint);
|
||||
}
|
||||
|
||||
// 写入 event vectors
|
||||
if (eventMetas.length > 0) {
|
||||
const eventVectorItems = eventMetas.map((meta, idx) => ({
|
||||
eventId: meta.eventId,
|
||||
vector: eventVectors[idx],
|
||||
}));
|
||||
await saveEventVectors(chatId, eventVectorItems, manifest.fingerprint);
|
||||
}
|
||||
|
||||
// 写入 state atoms
|
||||
if (stateAtoms.length > 0) {
|
||||
saveStateAtoms(stateAtoms);
|
||||
}
|
||||
|
||||
// Write state vectors (semantic + optional r-vector)
|
||||
if (stateVectorMetas.length > 0) {
|
||||
const stateVectorItems = stateVectorMetas.map((meta, idx) => ({
|
||||
atomId: meta.atomId,
|
||||
floor: meta.floor,
|
||||
vector: stateVectors[idx],
|
||||
rVector: (stateRVectors[idx] && (!hasRVectorMeta || meta.hasRVector)) ? stateRVectors[idx] : null,
|
||||
}));
|
||||
await saveStateVectors(chatId, stateVectorItems, manifest.fingerprint);
|
||||
}
|
||||
|
||||
// 更新 meta
|
||||
await updateMeta(chatId, {
|
||||
fingerprint: manifest.fingerprint,
|
||||
lastChunkFloor: manifest.lastChunkFloor,
|
||||
});
|
||||
|
||||
xbLog.info(MODULE_ID, `导入完成: ${chunkMetas.length} chunks, ${eventMetas.length} events, ${stateAtoms.length} state atoms`);
|
||||
|
||||
return {
|
||||
chunkCount: chunkMetas.length,
|
||||
eventCount: eventMetas.length,
|
||||
warnings,
|
||||
fingerprintMismatch,
|
||||
};
|
||||
}
|
||||
Reference in New Issue
Block a user