X/LittleWhiteBox
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor diffusion to r-sem edges with time window and add rVector I/O

Browse Source
This commit is contained in:
bielie
2026-02-15 00:36:21 +08:00
parent a787e58833
commit 5dd1d6b50b
6 changed files with 238 additions and 91 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
modules/story-summary/vector/storage/vector-io.js
View File

@@ -1,4 +1,4 @@
// ═══════════════════════════════════════════════════════════════════════════
// ═══════════════════════════════════════════════════════════════════════════
// Vector Import/Export
// 向量数据导入导出（当前 chatId 级别）
// ═══════════════════════════════════════════════════════════════════════════
@@ -30,7 +30,7 @@ import { getEngineFingerprint } from '../utils/embedder.js';
import { getVectorConfig } from '../../data/config.js';
const MODULE_ID = 'vector-io';
const EXPORT_VERSION = 1;
const EXPORT_VERSION = 2;
// ═══════════════════════════════════════════════════════════════════════════
// 工具函数
@@ -139,9 +139,15 @@ export async function exportVectors(onProgress) {
// state vectors
const sortedStateVectors = [...stateVectors].sort((a, b) => String(a.atomId).localeCompare(String(b.atomId)));
const stateVectorsOrdered = sortedStateVectors.map(v => v.vector);
const rDims = sortedStateVectors.find(v => v.rVector?.length)?.rVector?.length || dims;
const stateRVectorsOrdered = sortedStateVectors.map(v =>
v.rVector?.length ? v.rVector : new Array(rDims).fill(0)
);
const stateVectorsJsonl = sortedStateVectors.map(v => JSON.stringify({
atomId: v.atomId,
floor: v.floor,
hasRVector: !!(v.rVector?.length),
rDims: v.rVector?.length || 0,
})).join('\n');
// manifest
@@ -156,6 +162,8 @@ export async function exportVectors(onProgress) {
eventCount: sortedEventVectors.length,
stateAtomCount: stateAtoms.length,
stateVectorCount: stateVectors.length,
stateRVectorCount: sortedStateVectors.filter(v => v.rVector?.length).length,
rDims,
lastChunkFloor: meta.lastChunkFloor ?? -1,
};
@@ -173,6 +181,9 @@ export async function exportVectors(onProgress) {
'state_vectors.bin': stateVectorsOrdered.length
? float32ToBytes(stateVectorsOrdered, dims)
: new Uint8Array(0),
'state_r_vectors.bin': stateRVectorsOrdered.length
? float32ToBytes(stateRVectorsOrdered, rDims)
: new Uint8Array(0),
}, { level: 1 }); // 降低压缩级别，速度优先
onProgress?.('下载文件...');
@@ -226,7 +237,7 @@ export async function importVectors(file, onProgress) {
const manifest = JSON.parse(strFromU8(unzipped['manifest.json']));
if (manifest.version !== EXPORT_VERSION) {
if (![1, 2].includes(manifest.version)) {
throw new Error(`不支持的版本: ${manifest.version}`);
}
@@ -275,11 +286,17 @@ export async function importVectors(file, onProgress) {
const stateVectorsJsonl = unzipped['state_vectors.jsonl'] ? strFromU8(unzipped['state_vectors.jsonl']) : '';
const stateVectorMetas = stateVectorsJsonl.split('\n').filter(Boolean).map(line => JSON.parse(line));
// 解析 L0 state vectors
// Parse L0 semantic vectors
const stateVectorsBytes = unzipped['state_vectors.bin'];
const stateVectors = (stateVectorsBytes && stateVectorMetas.length)
? bytesToFloat32(stateVectorsBytes, manifest.dims)
: [];
// Parse optional L0 r-vectors (for diffusion r-sem edges)
const stateRVectorsBytes = unzipped['state_r_vectors.bin'];
const stateRVectors = (stateRVectorsBytes && stateVectorMetas.length)
? bytesToFloat32(stateRVectorsBytes, manifest.rDims || manifest.dims)
: [];
const hasRVectorMeta = stateVectorMetas.some(m => typeof m.hasRVector === 'boolean');
// 校验数量
if (chunkMetas.length !== chunkVectors.length) {
@@ -291,6 +308,9 @@ export async function importVectors(file, onProgress) {
if (stateVectorMetas.length !== stateVectors.length) {
throw new Error(`state 向量数量不匹配: 元数据 ${stateVectorMetas.length}, 向量 ${stateVectors.length}`);
}
if (stateRVectors.length > 0 && stateVectorMetas.length !== stateRVectors.length) {
throw new Error(`state r-vector count mismatch: meta=${stateVectorMetas.length}, vectors=${stateRVectors.length}`);
}
onProgress?.('清空旧数据...');
@@ -337,12 +357,13 @@ export async function importVectors(file, onProgress) {
saveStateAtoms(stateAtoms);
}
// 写入 state vectors
// Write state vectors (semantic + optional r-vector)
if (stateVectorMetas.length > 0) {
const stateVectorItems = stateVectorMetas.map((meta, idx) => ({
atomId: meta.atomId,
floor: meta.floor,
vector: stateVectors[idx],
rVector: (stateRVectors[idx] && (!hasRVectorMeta || meta.hasRVector)) ? stateRVectors[idx] : null,
}));
await saveStateVectors(chatId, stateVectorItems, manifest.fingerprint);
}