2.0变量 , 向量总结正式推送
This commit is contained in:
RT15548
928
modules/story-summary/vector/retrieval/diffusion.js
Normal file
928
modules/story-summary/vector/retrieval/diffusion.js
Normal file
@@ -0,0 +1,928 @@
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
// diffusion.js - PPR Graph Diffusion (Personalized PageRank)
|
||||
//
|
||||
// Spreads activation from seed L0 atoms through entity co-occurrence graph
|
||||
// to discover narratively-connected but semantically-distant memories.
|
||||
//
|
||||
// Pipeline position: recall.js Stage 7.5
|
||||
// Input: seeds (reranked L0 from Stage 6)
|
||||
// Output: additional L0 atoms → merged into l0Selected
|
||||
//
|
||||
// Algorithm:
|
||||
// 1. Build undirected weighted graph over all L0 atoms
|
||||
// Candidate edges: WHAT + R semantic; WHO/WHERE are reweight-only
|
||||
// 2. Personalized PageRank (Power Iteration)
|
||||
// Seeds weighted by rerankScore — Haveliwala (2002) topic-sensitive variant
|
||||
// α = 0.15 restart probability — Page et al. (1998)
|
||||
// 3. Post-verification (Dense Cosine Gate)
|
||||
// Exclude seeds, cosine ≥ 0.45, final = PPR_norm × cosine ≥ 0.10
|
||||
//
|
||||
// References:
|
||||
// Page et al. "The PageRank Citation Ranking" (1998)
|
||||
// Haveliwala "Topic-Sensitive PageRank" (IEEE TKDE 2003)
|
||||
// Langville & Meyer "Eigenvector Methods for Web IR" (SIAM Review 2005)
|
||||
// Sun et al. "GraftNet" (EMNLP 2018)
|
||||
// Jaccard "Étude comparative de la distribution florale" (1912)
|
||||
// Szymkiewicz "Une contribution statistique" (1934) — Overlap coefficient
|
||||
// Rimmon-Kenan "Narrative Fiction" (2002) — Channel weight rationale
|
||||
//
|
||||
// Core PPR iteration aligned with NetworkX pagerank():
|
||||
// github.com/networkx/networkx — algorithms/link_analysis/pagerank_alg.py
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
|
||||
import { xbLog } from '../../../../core/debug-core.js';
|
||||
import { getContext } from '../../../../../../../extensions.js';
|
||||
|
||||
const MODULE_ID = 'diffusion';
|
||||
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
// Configuration
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
|
||||
const CONFIG = {
|
||||
// PPR parameters (Page et al. 1998; GraftNet 2018 uses same values)
|
||||
ALPHA: 0.15, // restart probability
|
||||
EPSILON: 1e-5, // L1 convergence threshold
|
||||
MAX_ITER: 50, // hard iteration cap (typically converges in 15-25)
|
||||
|
||||
// Edge weight channel coefficients
|
||||
// Candidate generation uses WHAT + R semantic only.
|
||||
// WHO/WHERE are reweight-only signals.
|
||||
GAMMA: {
|
||||
what: 0.40, // interaction pair overlap
|
||||
rSem: 0.40, // semantic similarity over edges.r aggregate
|
||||
who: 0.10, // endpoint entity overlap (reweight-only)
|
||||
where: 0.05, // location exact match (reweight-only)
|
||||
time: 0.05, // temporal decay score
|
||||
},
|
||||
// R semantic candidate generation
|
||||
R_SEM_MIN_SIM: 0.62,
|
||||
R_SEM_TOPK: 8,
|
||||
TIME_WINDOW_MAX: 80,
|
||||
TIME_DECAY_DIVISOR: 12,
|
||||
WHERE_MAX_GROUP_SIZE: 16, // skip location-only pair expansion for over-common places
|
||||
WHERE_FREQ_DAMP_PIVOT: 6, // location freq <= pivot keeps full WHERE score
|
||||
WHERE_FREQ_DAMP_MIN: 0.20, // lower bound for damped WHERE contribution
|
||||
|
||||
// Post-verification (Cosine Gate)
|
||||
COSINE_GATE: 0.46, // min cosine(queryVector, stateVector)
|
||||
SCORE_FLOOR: 0.10, // min finalScore = PPR_normalized × cosine
|
||||
DIFFUSION_CAP: 100, // max diffused nodes (excluding seeds)
|
||||
};
|
||||
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
// Utility functions
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
|
||||
/**
|
||||
* Unicode-safe text normalization (matches recall.js / entity-lexicon.js)
|
||||
*/
|
||||
function normalize(s) {
|
||||
return String(s || '')
|
||||
.normalize('NFKC')
|
||||
.replace(/[\u200B-\u200D\uFEFF]/g, '')
|
||||
.trim()
|
||||
.toLowerCase();
|
||||
}
|
||||
|
||||
/**
|
||||
* Cosine similarity between two vectors
|
||||
*/
|
||||
function cosineSimilarity(a, b) {
|
||||
if (!a?.length || !b?.length || a.length !== b.length) return 0;
|
||||
let dot = 0, nA = 0, nB = 0;
|
||||
for (let i = 0; i < a.length; i++) {
|
||||
dot += a[i] * b[i];
|
||||
nA += a[i] * a[i];
|
||||
nB += b[i] * b[i];
|
||||
}
|
||||
return nA && nB ? dot / (Math.sqrt(nA) * Math.sqrt(nB)) : 0;
|
||||
}
|
||||
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
// Feature extraction from L0 atoms
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
|
||||
/**
|
||||
* Endpoint entity set from edges.s/edges.t (used for candidate pair generation).
|
||||
* @param {object} atom
|
||||
* @param {Set<string>} excludeEntities - entities to exclude (e.g. name1)
|
||||
* @returns {Set<string>}
|
||||
*/
|
||||
function extractEntities(atom, excludeEntities = new Set()) {
|
||||
const set = new Set();
|
||||
for (const e of (atom.edges || [])) {
|
||||
const s = normalize(e?.s);
|
||||
const t = normalize(e?.t);
|
||||
if (s && !excludeEntities.has(s)) set.add(s);
|
||||
if (t && !excludeEntities.has(t)) set.add(t);
|
||||
}
|
||||
return set;
|
||||
}
|
||||
|
||||
/**
|
||||
* WHAT channel: interaction pairs "A↔B" (direction-insensitive).
|
||||
* @param {object} atom
|
||||
* @param {Set<string>} excludeEntities
|
||||
* @returns {Set<string>}
|
||||
*/
|
||||
function extractInteractionPairs(atom, excludeEntities = new Set()) {
|
||||
const set = new Set();
|
||||
for (const e of (atom.edges || [])) {
|
||||
const s = normalize(e?.s);
|
||||
const t = normalize(e?.t);
|
||||
if (s && t && !excludeEntities.has(s) && !excludeEntities.has(t)) {
|
||||
const pair = [s, t].sort().join('\u2194');
|
||||
set.add(pair);
|
||||
}
|
||||
}
|
||||
return set;
|
||||
}
|
||||
|
||||
/**
|
||||
* WHERE channel: normalized location string
|
||||
* @param {object} atom
|
||||
* @returns {string} empty string if absent
|
||||
*/
|
||||
function extractLocation(atom) {
|
||||
return normalize(atom.where);
|
||||
}
|
||||
|
||||
function getFloorDistance(a, b) {
|
||||
const fa = Number(a?.floor || 0);
|
||||
const fb = Number(b?.floor || 0);
|
||||
return Math.abs(fa - fb);
|
||||
}
|
||||
|
||||
function getTimeScore(distance) {
|
||||
return Math.exp(-distance / CONFIG.TIME_DECAY_DIVISOR);
|
||||
}
|
||||
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
// Set similarity functions
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
|
||||
/**
|
||||
* Jaccard index: |A∩B| / |A∪B| (Jaccard 1912)
|
||||
* @param {Set<string>} a
|
||||
* @param {Set<string>} b
|
||||
* @returns {number} 0..1
|
||||
*/
|
||||
function jaccard(a, b) {
|
||||
if (!a.size || !b.size) return 0;
|
||||
let inter = 0;
|
||||
const [smaller, larger] = a.size <= b.size ? [a, b] : [b, a];
|
||||
for (const x of smaller) {
|
||||
if (larger.has(x)) inter++;
|
||||
}
|
||||
const union = a.size + b.size - inter;
|
||||
return union > 0 ? inter / union : 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Overlap coefficient: |A∩B| / min(|A|,|B|) (Szymkiewicz-Simpson 1934)
|
||||
* Used for directed pairs where set sizes are small (1-3); Jaccard
|
||||
* over-penalizes small-set asymmetry.
|
||||
* @param {Set<string>} a
|
||||
* @param {Set<string>} b
|
||||
* @returns {number} 0..1
|
||||
*/
|
||||
function overlapCoefficient(a, b) {
|
||||
if (!a.size || !b.size) return 0;
|
||||
let inter = 0;
|
||||
const [smaller, larger] = a.size <= b.size ? [a, b] : [b, a];
|
||||
for (const x of smaller) {
|
||||
if (larger.has(x)) inter++;
|
||||
}
|
||||
return inter / smaller.size;
|
||||
}
|
||||
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
// Graph construction
|
||||
//
|
||||
// Candidate pairs discovered via WHAT inverted index and R semantic top-k.
|
||||
// WHO/WHERE are reweight-only signals and never create candidate pairs.
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
|
||||
/**
|
||||
* Pre-extract features for all atoms
|
||||
* @param {object[]} allAtoms
|
||||
* @param {Set<string>} excludeEntities
|
||||
* @returns {object[]} feature objects with entities/interactionPairs/location
|
||||
*/
|
||||
function extractAllFeatures(allAtoms, excludeEntities = new Set()) {
|
||||
return allAtoms.map(atom => ({
|
||||
entities: extractEntities(atom, excludeEntities),
|
||||
interactionPairs: extractInteractionPairs(atom, excludeEntities),
|
||||
location: extractLocation(atom),
|
||||
}));
|
||||
}
|
||||
|
||||
/**
|
||||
* Build inverted index: value → list of atom indices
|
||||
* @param {object[]} features
|
||||
* @returns {{ whatIndex: Map, locationFreq: Map }}
|
||||
*/
|
||||
function buildInvertedIndices(features) {
|
||||
const whatIndex = new Map();
|
||||
const locationFreq = new Map();
|
||||
|
||||
for (let i = 0; i < features.length; i++) {
|
||||
for (const pair of features[i].interactionPairs) {
|
||||
if (!whatIndex.has(pair)) whatIndex.set(pair, []);
|
||||
whatIndex.get(pair).push(i);
|
||||
}
|
||||
const loc = features[i].location;
|
||||
if (loc) locationFreq.set(loc, (locationFreq.get(loc) || 0) + 1);
|
||||
}
|
||||
|
||||
return { whatIndex, locationFreq };
|
||||
}
|
||||
|
||||
/**
|
||||
* Collect candidate pairs from inverted index
|
||||
* @param {Map} index - value → [atomIndex, ...]
|
||||
* @param {Set<number>} pairSet - packed pair collector
|
||||
* @param {number} N - total atom count (for pair packing)
|
||||
*/
|
||||
function collectPairsFromIndex(index, pairSet, N) {
|
||||
for (const indices of index.values()) {
|
||||
for (let a = 0; a < indices.length; a++) {
|
||||
for (let b = a + 1; b < indices.length; b++) {
|
||||
const lo = Math.min(indices[a], indices[b]);
|
||||
const hi = Math.max(indices[a], indices[b]);
|
||||
pairSet.add(lo * N + hi);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Build weighted undirected graph over L0 atoms.
|
||||
*
|
||||
* @param {object[]} allAtoms
|
||||
* @param {object[]} stateVectors
|
||||
* @param {Set<string>} excludeEntities
|
||||
* @returns {{ neighbors: object[][], edgeCount: number, channelStats: object, buildTime: number }}
|
||||
*/
|
||||
function buildGraph(allAtoms, stateVectors = [], excludeEntities = new Set()) {
|
||||
const N = allAtoms.length;
|
||||
const T0 = performance.now();
|
||||
|
||||
const features = extractAllFeatures(allAtoms, excludeEntities);
|
||||
const { whatIndex, locationFreq } = buildInvertedIndices(features);
|
||||
|
||||
// Candidate pairs: WHAT + R semantic
|
||||
const pairSetByWhat = new Set();
|
||||
const pairSetByRSem = new Set();
|
||||
const rSemByPair = new Map();
|
||||
const pairSet = new Set();
|
||||
collectPairsFromIndex(whatIndex, pairSetByWhat, N);
|
||||
|
||||
const rVectorByAtomId = new Map(
|
||||
(stateVectors || [])
|
||||
.filter(v => v?.atomId && v?.rVector?.length)
|
||||
.map(v => [v.atomId, v.rVector])
|
||||
);
|
||||
const rVectors = allAtoms.map(a => rVectorByAtomId.get(a.atomId) || null);
|
||||
|
||||
const directedNeighbors = Array.from({ length: N }, () => []);
|
||||
let rSemSimSum = 0;
|
||||
let rSemSimCount = 0;
|
||||
let topKPrunedPairs = 0;
|
||||
let timeWindowFilteredPairs = 0;
|
||||
|
||||
// Enumerate only pairs within floor window to avoid O(N^2) full scan.
|
||||
const sortedByFloor = allAtoms
|
||||
.map((atom, idx) => ({ idx, floor: Number(atom?.floor || 0) }))
|
||||
.sort((a, b) => a.floor - b.floor);
|
||||
|
||||
for (let left = 0; left < sortedByFloor.length; left++) {
|
||||
const i = sortedByFloor[left].idx;
|
||||
const baseFloor = sortedByFloor[left].floor;
|
||||
|
||||
for (let right = left + 1; right < sortedByFloor.length; right++) {
|
||||
const floorDelta = sortedByFloor[right].floor - baseFloor;
|
||||
if (floorDelta > CONFIG.TIME_WINDOW_MAX) break;
|
||||
|
||||
const j = sortedByFloor[right].idx;
|
||||
const vi = rVectors[i];
|
||||
const vj = rVectors[j];
|
||||
if (!vi?.length || !vj?.length) continue;
|
||||
|
||||
const sim = cosineSimilarity(vi, vj);
|
||||
if (sim < CONFIG.R_SEM_MIN_SIM) continue;
|
||||
|
||||
directedNeighbors[i].push({ target: j, sim });
|
||||
directedNeighbors[j].push({ target: i, sim });
|
||||
rSemSimSum += sim;
|
||||
rSemSimCount++;
|
||||
}
|
||||
}
|
||||
|
||||
for (let i = 0; i < N; i++) {
|
||||
const arr = directedNeighbors[i];
|
||||
if (!arr.length) continue;
|
||||
arr.sort((a, b) => b.sim - a.sim);
|
||||
if (arr.length > CONFIG.R_SEM_TOPK) {
|
||||
topKPrunedPairs += arr.length - CONFIG.R_SEM_TOPK;
|
||||
}
|
||||
for (const n of arr.slice(0, CONFIG.R_SEM_TOPK)) {
|
||||
const lo = Math.min(i, n.target);
|
||||
const hi = Math.max(i, n.target);
|
||||
const packed = lo * N + hi;
|
||||
pairSetByRSem.add(packed);
|
||||
const prev = rSemByPair.get(packed) || 0;
|
||||
if (n.sim > prev) rSemByPair.set(packed, n.sim);
|
||||
}
|
||||
}
|
||||
for (const p of pairSetByWhat) pairSet.add(p);
|
||||
for (const p of pairSetByRSem) pairSet.add(p);
|
||||
|
||||
// Compute edge weights for all candidates
|
||||
const neighbors = Array.from({ length: N }, () => []);
|
||||
let edgeCount = 0;
|
||||
const channelStats = { what: 0, where: 0, rSem: 0, who: 0 };
|
||||
let reweightWhoUsed = 0;
|
||||
let reweightWhereUsed = 0;
|
||||
|
||||
for (const packed of pairSet) {
|
||||
const i = Math.floor(packed / N);
|
||||
const j = packed % N;
|
||||
|
||||
const distance = getFloorDistance(allAtoms[i], allAtoms[j]);
|
||||
if (distance > CONFIG.TIME_WINDOW_MAX) {
|
||||
timeWindowFilteredPairs++;
|
||||
continue;
|
||||
}
|
||||
const wTime = getTimeScore(distance);
|
||||
|
||||
const fi = features[i];
|
||||
const fj = features[j];
|
||||
|
||||
const wWhat = overlapCoefficient(fi.interactionPairs, fj.interactionPairs);
|
||||
const wRSem = rSemByPair.get(packed) || 0;
|
||||
const wWho = jaccard(fi.entities, fj.entities);
|
||||
let wWhere = 0.0;
|
||||
if (fi.location && fi.location === fj.location) {
|
||||
const freq = locationFreq.get(fi.location) || 1;
|
||||
const damp = Math.max(
|
||||
CONFIG.WHERE_FREQ_DAMP_MIN,
|
||||
Math.min(1, CONFIG.WHERE_FREQ_DAMP_PIVOT / Math.max(1, freq))
|
||||
);
|
||||
wWhere = damp;
|
||||
}
|
||||
|
||||
const weight =
|
||||
CONFIG.GAMMA.what * wWhat +
|
||||
CONFIG.GAMMA.rSem * wRSem +
|
||||
CONFIG.GAMMA.who * wWho +
|
||||
CONFIG.GAMMA.where * wWhere +
|
||||
CONFIG.GAMMA.time * wTime;
|
||||
|
||||
if (weight > 0) {
|
||||
neighbors[i].push({ target: j, weight });
|
||||
neighbors[j].push({ target: i, weight });
|
||||
edgeCount++;
|
||||
|
||||
if (wWhat > 0) channelStats.what++;
|
||||
if (wRSem > 0) channelStats.rSem++;
|
||||
if (wWho > 0) channelStats.who++;
|
||||
if (wWhere > 0) channelStats.where++;
|
||||
if (wWho > 0) reweightWhoUsed++;
|
||||
if (wWhere > 0) reweightWhereUsed++;
|
||||
}
|
||||
}
|
||||
|
||||
const buildTime = Math.round(performance.now() - T0);
|
||||
|
||||
xbLog.info(MODULE_ID,
|
||||
`Graph: ${N} nodes, ${edgeCount} edges ` +
|
||||
`(candidate_by_what=${pairSetByWhat.size} candidate_by_r_sem=${pairSetByRSem.size}) ` +
|
||||
`(what=${channelStats.what} r_sem=${channelStats.rSem} who=${channelStats.who} where=${channelStats.where}) ` +
|
||||
`(reweight_who_used=${reweightWhoUsed} reweight_where_used=${reweightWhereUsed}) ` +
|
||||
`(time_window_filtered=${timeWindowFilteredPairs} topk_pruned=${topKPrunedPairs}) ` +
|
||||
`(${buildTime}ms)`
|
||||
);
|
||||
|
||||
const totalPairs = N > 1 ? (N * (N - 1)) / 2 : 0;
|
||||
const edgeDensity = totalPairs > 0 ? Number((edgeCount / totalPairs * 100).toFixed(2)) : 0;
|
||||
|
||||
return {
|
||||
neighbors,
|
||||
edgeCount,
|
||||
channelStats,
|
||||
buildTime,
|
||||
candidatePairs: pairSet.size,
|
||||
pairsFromWhat: pairSetByWhat.size,
|
||||
pairsFromRSem: pairSetByRSem.size,
|
||||
rSemAvgSim: rSemSimCount ? Number((rSemSimSum / rSemSimCount).toFixed(3)) : 0,
|
||||
timeWindowFilteredPairs,
|
||||
topKPrunedPairs,
|
||||
reweightWhoUsed,
|
||||
reweightWhereUsed,
|
||||
edgeDensity,
|
||||
};
|
||||
}
|
||||
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
// PPR: Seed vector construction
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
|
||||
/**
|
||||
* Build personalization vector s from seeds, weighted by rerankScore.
|
||||
* Haveliwala (2002): non-uniform personalization improves topic sensitivity.
|
||||
*
|
||||
* @param {object[]} seeds - seed L0 entries with atomId and rerankScore
|
||||
* @param {Map<string, number>} idToIdx - atomId → array index
|
||||
* @param {number} N - total node count
|
||||
* @returns {Float64Array} personalization vector (L1-normalized, sums to 1)
|
||||
*/
|
||||
function buildSeedVector(seeds, idToIdx, N) {
|
||||
const s = new Float64Array(N);
|
||||
let total = 0;
|
||||
|
||||
for (const seed of seeds) {
|
||||
const idx = idToIdx.get(seed.atomId);
|
||||
if (idx == null) continue;
|
||||
|
||||
const score = Math.max(0, seed.rerankScore || seed.similarity || 0);
|
||||
s[idx] += score;
|
||||
total += score;
|
||||
}
|
||||
|
||||
// L1 normalize to probability distribution
|
||||
if (total > 0) {
|
||||
for (let i = 0; i < N; i++) s[i] /= total;
|
||||
}
|
||||
|
||||
return s;
|
||||
}
|
||||
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
// PPR: Column normalization + dangling node detection
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
|
||||
/**
|
||||
* Column-normalize adjacency into transition matrix W.
|
||||
*
|
||||
* Column j of W: W_{ij} = weight(i,j) / Σ_k weight(k,j)
|
||||
* Dangling nodes (no outgoing edges): handled in powerIteration
|
||||
* via redistribution to personalization vector s.
|
||||
* (Langville & Meyer 2005, §4.1)
|
||||
*
|
||||
* @param {object[][]} neighbors - neighbors[j] = [{target, weight}, ...]
|
||||
* @param {number} N
|
||||
* @returns {{ columns: object[][], dangling: number[] }}
|
||||
*/
|
||||
function columnNormalize(neighbors, N) {
|
||||
const columns = Array.from({ length: N }, () => []);
|
||||
const dangling = [];
|
||||
|
||||
for (let j = 0; j < N; j++) {
|
||||
const edges = neighbors[j];
|
||||
|
||||
let sum = 0;
|
||||
for (let e = 0; e < edges.length; e++) sum += edges[e].weight;
|
||||
|
||||
if (sum <= 0) {
|
||||
dangling.push(j);
|
||||
continue;
|
||||
}
|
||||
|
||||
const col = columns[j];
|
||||
for (let e = 0; e < edges.length; e++) {
|
||||
col.push({ target: edges[e].target, prob: edges[e].weight / sum });
|
||||
}
|
||||
}
|
||||
|
||||
return { columns, dangling };
|
||||
}
|
||||
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
// PPR: Power Iteration
|
||||
//
|
||||
// Aligned with NetworkX pagerank() (pagerank_alg.py):
|
||||
//
|
||||
// NetworkX "alpha" = damping = our (1 − α)
|
||||
// NetworkX "1-alpha" = teleportation = our α
|
||||
//
|
||||
// Per iteration:
|
||||
// π_new[i] = α·s[i] + (1−α)·( Σ_j W_{ij}·π[j] + dangling_sum·s[i] )
|
||||
//
|
||||
// Convergence: Perron-Frobenius theorem guarantees unique stationary
|
||||
// distribution for irreducible aperiodic column-stochastic matrix.
|
||||
// Rate: ‖π^(t+1) − π^t‖₁ ≤ (1−α)^t (geometric).
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
|
||||
/**
|
||||
* Run PPR Power Iteration.
|
||||
*
|
||||
* @param {object[][]} columns - column-normalized transition matrix
|
||||
* @param {Float64Array} s - personalization vector (sums to 1)
|
||||
* @param {number[]} dangling - dangling node indices
|
||||
* @param {number} N - node count
|
||||
* @returns {{ pi: Float64Array, iterations: number, finalError: number }}
|
||||
*/
|
||||
function powerIteration(columns, s, dangling, N) {
|
||||
const alpha = CONFIG.ALPHA;
|
||||
const d = 1 - alpha; // damping factor = prob of following edges
|
||||
const epsilon = CONFIG.EPSILON;
|
||||
const maxIter = CONFIG.MAX_ITER;
|
||||
|
||||
// Initialize π to personalization vector
|
||||
let pi = new Float64Array(N);
|
||||
for (let i = 0; i < N; i++) pi[i] = s[i];
|
||||
|
||||
let iterations = 0;
|
||||
let finalError = 0;
|
||||
|
||||
for (let iter = 0; iter < maxIter; iter++) {
|
||||
const piNew = new Float64Array(N);
|
||||
|
||||
// Dangling mass: probability at nodes with no outgoing edges
|
||||
// redistributed to personalization vector (Langville & Meyer 2005)
|
||||
let danglingSum = 0;
|
||||
for (let k = 0; k < dangling.length; k++) {
|
||||
danglingSum += pi[dangling[k]];
|
||||
}
|
||||
|
||||
// Sparse matrix-vector product: (1−α) · W · π
|
||||
for (let j = 0; j < N; j++) {
|
||||
const pj = pi[j];
|
||||
if (pj === 0) continue;
|
||||
|
||||
const col = columns[j];
|
||||
const dpj = d * pj;
|
||||
for (let e = 0; e < col.length; e++) {
|
||||
piNew[col[e].target] += dpj * col[e].prob;
|
||||
}
|
||||
}
|
||||
|
||||
// Restart + dangling contribution:
|
||||
// α · s[i] + (1−α) · danglingSum · s[i]
|
||||
const restartCoeff = alpha + d * danglingSum;
|
||||
for (let i = 0; i < N; i++) {
|
||||
piNew[i] += restartCoeff * s[i];
|
||||
}
|
||||
|
||||
// L1 convergence check
|
||||
let l1 = 0;
|
||||
for (let i = 0; i < N; i++) {
|
||||
l1 += Math.abs(piNew[i] - pi[i]);
|
||||
}
|
||||
|
||||
pi = piNew;
|
||||
iterations = iter + 1;
|
||||
finalError = l1;
|
||||
|
||||
if (l1 < epsilon) break;
|
||||
}
|
||||
|
||||
return { pi, iterations, finalError };
|
||||
}
|
||||
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
// Post-verification: Dense Cosine Gate
|
||||
//
|
||||
// PPR measures graph-structural relevance ("same characters").
|
||||
// Cosine gate measures semantic relevance ("related to current topic").
|
||||
// Product combination ensures both dimensions are satisfied
|
||||
// (CombMNZ — Fox & Shaw, TREC-2 1994).
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
|
||||
/**
|
||||
* Filter PPR-activated nodes by semantic relevance.
|
||||
*
|
||||
* For each non-seed node with PPR > 0:
|
||||
* 1. cosine(queryVector, stateVector) ≥ COSINE_GATE
|
||||
* 2. finalScore = PPR_normalized × cosine ≥ SCORE_FLOOR
|
||||
* 3. Top DIFFUSION_CAP by finalScore
|
||||
*
|
||||
* @param {Float64Array} pi - PPR stationary distribution
|
||||
* @param {string[]} atomIds - index → atomId
|
||||
* @param {Map<string, object>} atomById - atomId → atom object
|
||||
* @param {Set<string>} seedAtomIds - seed atomIds (excluded from output)
|
||||
* @param {Map<string, Float32Array>} vectorMap - atomId → embedding vector
|
||||
* @param {Float32Array|number[]} queryVector - R2 weighted query vector
|
||||
* @returns {{ diffused: object[], gateStats: object }}
|
||||
*/
|
||||
function postVerify(pi, atomIds, atomById, seedAtomIds, vectorMap, queryVector) {
|
||||
const N = atomIds.length;
|
||||
const gateStats = { passed: 0, filtered: 0, noVector: 0 };
|
||||
|
||||
// Find max PPR score among non-seed nodes (for normalization)
|
||||
let maxPPR = 0;
|
||||
for (let i = 0; i < N; i++) {
|
||||
if (pi[i] > 0 && !seedAtomIds.has(atomIds[i])) {
|
||||
if (pi[i] > maxPPR) maxPPR = pi[i];
|
||||
}
|
||||
}
|
||||
|
||||
if (maxPPR <= 0) {
|
||||
return { diffused: [], gateStats };
|
||||
}
|
||||
|
||||
const candidates = [];
|
||||
|
||||
for (let i = 0; i < N; i++) {
|
||||
const atomId = atomIds[i];
|
||||
|
||||
// Skip seeds and zero-probability nodes
|
||||
if (seedAtomIds.has(atomId)) continue;
|
||||
if (pi[i] <= 0) continue;
|
||||
|
||||
// Require state vector for cosine verification
|
||||
const vec = vectorMap.get(atomId);
|
||||
if (!vec?.length) {
|
||||
gateStats.noVector++;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Cosine gate
|
||||
const cos = cosineSimilarity(queryVector, vec);
|
||||
if (cos < CONFIG.COSINE_GATE) {
|
||||
gateStats.filtered++;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Final score = PPR_normalized × cosine
|
||||
const pprNorm = pi[i] / maxPPR;
|
||||
const finalScore = pprNorm * cos;
|
||||
|
||||
if (finalScore < CONFIG.SCORE_FLOOR) {
|
||||
gateStats.filtered++;
|
||||
continue;
|
||||
}
|
||||
|
||||
gateStats.passed++;
|
||||
|
||||
const atom = atomById.get(atomId);
|
||||
if (!atom) continue;
|
||||
|
||||
candidates.push({
|
||||
atomId,
|
||||
floor: atom.floor,
|
||||
atom,
|
||||
finalScore,
|
||||
pprScore: pi[i],
|
||||
pprNormalized: pprNorm,
|
||||
cosine: cos,
|
||||
});
|
||||
}
|
||||
|
||||
// Sort by finalScore descending, cap at DIFFUSION_CAP
|
||||
candidates.sort((a, b) => b.finalScore - a.finalScore);
|
||||
const diffused = candidates.slice(0, CONFIG.DIFFUSION_CAP);
|
||||
|
||||
return { diffused, gateStats };
|
||||
}
|
||||
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
// Main entry point
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
|
||||
/**
|
||||
* Spread activation from seed L0 atoms through entity co-occurrence graph.
|
||||
*
|
||||
* Called from recall.js Stage 7.5, after locateAndPullEvidence and before
|
||||
* Causation Trace. Results are merged into l0Selected and consumed by
|
||||
* prompt.js through existing budget/formatting pipeline (zero downstream changes).
|
||||
*
|
||||
* @param {object[]} seeds - l0Selected from recall Stage 6
|
||||
* Each: { atomId, rerankScore, similarity, atom, ... }
|
||||
* @param {object[]} allAtoms - getStateAtoms() result
|
||||
* Each: { atomId, floor, semantic, edges, where }
|
||||
* @param {object[]} stateVectors - getAllStateVectors() result
|
||||
* Each: { atomId, floor, vector: Float32Array, rVector?: Float32Array }
|
||||
* @param {Float32Array|number[]} queryVector - R2 weighted query vector
|
||||
* @param {object|null} metrics - metrics object (optional, mutated in-place)
|
||||
* @returns {object[]} Additional L0 atoms for l0Selected
|
||||
* Each: { atomId, floor, atom, finalScore, pprScore, pprNormalized, cosine }
|
||||
*/
|
||||
export function diffuseFromSeeds(seeds, allAtoms, stateVectors, queryVector, metrics) {
|
||||
const T0 = performance.now();
|
||||
|
||||
// ─── Early exits ─────────────────────────────────────────────────
|
||||
|
||||
if (!seeds?.length || !allAtoms?.length || !queryVector?.length) {
|
||||
fillMetricsEmpty(metrics);
|
||||
return [];
|
||||
}
|
||||
|
||||
// Align with entity-lexicon hard rule: exclude name1 from graph features.
|
||||
const { name1 } = getContext();
|
||||
const excludeEntities = new Set();
|
||||
if (name1) excludeEntities.add(normalize(name1));
|
||||
|
||||
// ─── 1. Build atom index ─────────────────────────────────────────
|
||||
|
||||
const atomById = new Map();
|
||||
const atomIds = [];
|
||||
const idToIdx = new Map();
|
||||
|
||||
for (let i = 0; i < allAtoms.length; i++) {
|
||||
const a = allAtoms[i];
|
||||
atomById.set(a.atomId, a);
|
||||
atomIds.push(a.atomId);
|
||||
idToIdx.set(a.atomId, i);
|
||||
}
|
||||
|
||||
const N = allAtoms.length;
|
||||
|
||||
// Validate seeds against atom index
|
||||
const validSeeds = seeds.filter(s => idToIdx.has(s.atomId));
|
||||
const seedAtomIds = new Set(validSeeds.map(s => s.atomId));
|
||||
|
||||
if (!validSeeds.length) {
|
||||
fillMetricsEmpty(metrics);
|
||||
return [];
|
||||
}
|
||||
|
||||
// ─── 2. Build graph ──────────────────────────────────────────────
|
||||
|
||||
const graph = buildGraph(allAtoms, stateVectors, excludeEntities);
|
||||
|
||||
if (graph.edgeCount === 0) {
|
||||
fillMetrics(metrics, {
|
||||
seedCount: validSeeds.length,
|
||||
graphNodes: N,
|
||||
graphEdges: 0,
|
||||
channelStats: graph.channelStats,
|
||||
candidatePairs: graph.candidatePairs,
|
||||
pairsFromWhat: graph.pairsFromWhat,
|
||||
pairsFromRSem: graph.pairsFromRSem,
|
||||
rSemAvgSim: graph.rSemAvgSim,
|
||||
timeWindowFilteredPairs: graph.timeWindowFilteredPairs,
|
||||
topKPrunedPairs: graph.topKPrunedPairs,
|
||||
edgeDensity: graph.edgeDensity,
|
||||
reweightWhoUsed: graph.reweightWhoUsed,
|
||||
reweightWhereUsed: graph.reweightWhereUsed,
|
||||
time: graph.buildTime,
|
||||
});
|
||||
xbLog.info(MODULE_ID, 'No graph edges — skipping diffusion');
|
||||
return [];
|
||||
}
|
||||
|
||||
// ─── 3. Build seed vector ────────────────────────────────────────
|
||||
|
||||
const s = buildSeedVector(validSeeds, idToIdx, N);
|
||||
|
||||
// ─── 4. Column normalize ─────────────────────────────────────────
|
||||
|
||||
const { columns, dangling } = columnNormalize(graph.neighbors, N);
|
||||
|
||||
// ─── 5. PPR Power Iteration ──────────────────────────────────────
|
||||
|
||||
const T_PPR = performance.now();
|
||||
const { pi, iterations, finalError } = powerIteration(columns, s, dangling, N);
|
||||
const pprTime = Math.round(performance.now() - T_PPR);
|
||||
|
||||
// Count activated non-seed nodes
|
||||
let pprActivated = 0;
|
||||
for (let i = 0; i < N; i++) {
|
||||
if (pi[i] > 0 && !seedAtomIds.has(atomIds[i])) pprActivated++;
|
||||
}
|
||||
|
||||
// ─── 6. Post-verification ────────────────────────────────────────
|
||||
|
||||
const vectorMap = new Map();
|
||||
for (const sv of (stateVectors || [])) {
|
||||
vectorMap.set(sv.atomId, sv.vector);
|
||||
}
|
||||
|
||||
const { diffused, gateStats } = postVerify(
|
||||
pi, atomIds, atomById, seedAtomIds, vectorMap, queryVector
|
||||
);
|
||||
|
||||
// ─── 7. Metrics ──────────────────────────────────────────────────
|
||||
|
||||
const totalTime = Math.round(performance.now() - T0);
|
||||
|
||||
fillMetrics(metrics, {
|
||||
seedCount: validSeeds.length,
|
||||
graphNodes: N,
|
||||
graphEdges: graph.edgeCount,
|
||||
channelStats: graph.channelStats,
|
||||
candidatePairs: graph.candidatePairs,
|
||||
pairsFromWhat: graph.pairsFromWhat,
|
||||
pairsFromRSem: graph.pairsFromRSem,
|
||||
rSemAvgSim: graph.rSemAvgSim,
|
||||
timeWindowFilteredPairs: graph.timeWindowFilteredPairs,
|
||||
topKPrunedPairs: graph.topKPrunedPairs,
|
||||
edgeDensity: graph.edgeDensity,
|
||||
reweightWhoUsed: graph.reweightWhoUsed,
|
||||
reweightWhereUsed: graph.reweightWhereUsed,
|
||||
buildTime: graph.buildTime,
|
||||
iterations,
|
||||
convergenceError: finalError,
|
||||
pprActivated,
|
||||
cosineGatePassed: gateStats.passed,
|
||||
cosineGateFiltered: gateStats.filtered,
|
||||
cosineGateNoVector: gateStats.noVector,
|
||||
postGatePassRate: pprActivated > 0
|
||||
? Math.round((gateStats.passed / pprActivated) * 100)
|
||||
: 0,
|
||||
finalCount: diffused.length,
|
||||
scoreDistribution: diffused.length > 0
|
||||
? calcScoreStats(diffused.map(d => d.finalScore))
|
||||
: { min: 0, max: 0, mean: 0 },
|
||||
time: totalTime,
|
||||
});
|
||||
|
||||
xbLog.info(MODULE_ID,
|
||||
`Diffusion: ${validSeeds.length} seeds → ` +
|
||||
`graph(${N}n/${graph.edgeCount}e) → ` +
|
||||
`PPR(${iterations}it, ε=${finalError.toExponential(1)}, ${pprTime}ms) → ` +
|
||||
`${pprActivated} activated → ` +
|
||||
`gate(${gateStats.passed}\u2713/${gateStats.filtered}\u2717` +
|
||||
`${gateStats.noVector ? `/${gateStats.noVector}?` : ''}) → ` +
|
||||
`${diffused.length} final (${totalTime}ms)`
|
||||
);
|
||||
|
||||
return diffused;
|
||||
}
|
||||
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
// Metrics helpers
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
|
||||
/**
|
||||
* Compute min/max/mean distribution
|
||||
* @param {number[]} scores
|
||||
* @returns {{ min: number, max: number, mean: number }}
|
||||
*/
|
||||
function calcScoreStats(scores) {
|
||||
if (!scores.length) return { min: 0, max: 0, mean: 0 };
|
||||
const sorted = [...scores].sort((a, b) => a - b);
|
||||
const sum = sorted.reduce((a, b) => a + b, 0);
|
||||
return {
|
||||
min: Number(sorted[0].toFixed(3)),
|
||||
max: Number(sorted[sorted.length - 1].toFixed(3)),
|
||||
mean: Number((sum / sorted.length).toFixed(3)),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Fill metrics with empty diffusion block
|
||||
*/
|
||||
function fillMetricsEmpty(metrics) {
|
||||
if (!metrics) return;
|
||||
metrics.diffusion = {
|
||||
seedCount: 0,
|
||||
graphNodes: 0,
|
||||
graphEdges: 0,
|
||||
iterations: 0,
|
||||
convergenceError: 0,
|
||||
pprActivated: 0,
|
||||
cosineGatePassed: 0,
|
||||
cosineGateFiltered: 0,
|
||||
cosineGateNoVector: 0,
|
||||
finalCount: 0,
|
||||
scoreDistribution: { min: 0, max: 0, mean: 0 },
|
||||
byChannel: { what: 0, where: 0, rSem: 0, who: 0 },
|
||||
candidatePairs: 0,
|
||||
pairsFromWhat: 0,
|
||||
pairsFromRSem: 0,
|
||||
rSemAvgSim: 0,
|
||||
timeWindowFilteredPairs: 0,
|
||||
topKPrunedPairs: 0,
|
||||
edgeDensity: 0,
|
||||
reweightWhoUsed: 0,
|
||||
reweightWhereUsed: 0,
|
||||
postGatePassRate: 0,
|
||||
time: 0,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Fill metrics with diffusion results
|
||||
*/
|
||||
function fillMetrics(metrics, data) {
|
||||
if (!metrics) return;
|
||||
metrics.diffusion = {
|
||||
seedCount: data.seedCount || 0,
|
||||
graphNodes: data.graphNodes || 0,
|
||||
graphEdges: data.graphEdges || 0,
|
||||
iterations: data.iterations || 0,
|
||||
convergenceError: data.convergenceError || 0,
|
||||
pprActivated: data.pprActivated || 0,
|
||||
cosineGatePassed: data.cosineGatePassed || 0,
|
||||
cosineGateFiltered: data.cosineGateFiltered || 0,
|
||||
cosineGateNoVector: data.cosineGateNoVector || 0,
|
||||
postGatePassRate: data.postGatePassRate || 0,
|
||||
finalCount: data.finalCount || 0,
|
||||
scoreDistribution: data.scoreDistribution || { min: 0, max: 0, mean: 0 },
|
||||
byChannel: data.channelStats || { what: 0, where: 0, rSem: 0, who: 0 },
|
||||
candidatePairs: data.candidatePairs || 0,
|
||||
pairsFromWhat: data.pairsFromWhat || 0,
|
||||
pairsFromRSem: data.pairsFromRSem || 0,
|
||||
rSemAvgSim: data.rSemAvgSim || 0,
|
||||
timeWindowFilteredPairs: data.timeWindowFilteredPairs || 0,
|
||||
topKPrunedPairs: data.topKPrunedPairs || 0,
|
||||
edgeDensity: data.edgeDensity || 0,
|
||||
reweightWhoUsed: data.reweightWhoUsed || 0,
|
||||
reweightWhereUsed: data.reweightWhereUsed || 0,
|
||||
time: data.time || 0,
|
||||
};
|
||||
}
|
||||
221
modules/story-summary/vector/retrieval/entity-lexicon.js
Normal file
221
modules/story-summary/vector/retrieval/entity-lexicon.js
Normal file
@@ -0,0 +1,221 @@
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
// entity-lexicon.js - 实体词典（确定性，无 LLM）
|
||||
//
|
||||
// 职责：
|
||||
// 1. 从已有结构化存储构建可信实体词典
|
||||
// 2. 从文本中提取命中的实体
|
||||
//
|
||||
// 硬约束：name1 永不进入词典
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
|
||||
import { getStateAtoms } from '../storage/state-store.js';
|
||||
|
||||
// 人名词典黑名单：代词、标签词、明显非人物词
|
||||
const PERSON_LEXICON_BLACKLIST = new Set([
|
||||
'我', '你', '他', '她', '它', '我们', '你们', '他们', '她们', '它们',
|
||||
'自己', '对方', '用户', '助手', 'user', 'assistant',
|
||||
'男人', '女性', '成熟女性', '主人', '主角',
|
||||
'龟头', '子宫', '阴道', '阴茎',
|
||||
'电脑', '电脑屏幕', '手机', '监控画面', '摄像头', '阳光', '折叠床', '书房', '卫生间隔间',
|
||||
]);
|
||||
|
||||
/**
|
||||
* 标准化字符串（用于实体匹配）
|
||||
* @param {string} s
|
||||
* @returns {string}
|
||||
*/
|
||||
function normalize(s) {
|
||||
return String(s || '')
|
||||
.normalize('NFKC')
|
||||
.replace(/[\u200B-\u200D\uFEFF]/g, '')
|
||||
.trim()
|
||||
.toLowerCase();
|
||||
}
|
||||
|
||||
function isBlacklistedPersonTerm(raw) {
|
||||
return PERSON_LEXICON_BLACKLIST.has(normalize(raw));
|
||||
}
|
||||
|
||||
function addPersonTerm(set, raw) {
|
||||
const n = normalize(raw);
|
||||
if (!n || n.length < 2) return;
|
||||
if (isBlacklistedPersonTerm(n)) return;
|
||||
set.add(n);
|
||||
}
|
||||
|
||||
function collectTrustedCharacters(store, context) {
|
||||
const trusted = new Set();
|
||||
|
||||
const main = store?.json?.characters?.main || [];
|
||||
for (const m of main) {
|
||||
addPersonTerm(trusted, typeof m === 'string' ? m : m.name);
|
||||
}
|
||||
|
||||
const arcs = store?.json?.arcs || [];
|
||||
for (const a of arcs) {
|
||||
addPersonTerm(trusted, a.name);
|
||||
}
|
||||
|
||||
if (context?.name2) {
|
||||
addPersonTerm(trusted, context.name2);
|
||||
}
|
||||
|
||||
const events = store?.json?.events || [];
|
||||
for (const ev of events) {
|
||||
for (const p of (ev?.participants || [])) {
|
||||
addPersonTerm(trusted, p);
|
||||
}
|
||||
}
|
||||
|
||||
if (context?.name1) {
|
||||
trusted.delete(normalize(context.name1));
|
||||
}
|
||||
|
||||
return trusted;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build trusted character pool only (without scanning L0 candidate atoms).
|
||||
* trustedCharacters: main/arcs/name2/L2 participants, excludes name1.
|
||||
*
|
||||
* @param {object} store
|
||||
* @param {object} context
|
||||
* @returns {Set<string>}
|
||||
*/
|
||||
export function buildTrustedCharacters(store, context) {
|
||||
return collectTrustedCharacters(store, context);
|
||||
}
|
||||
|
||||
function collectCandidateCharactersFromL0(context) {
|
||||
const candidate = new Set();
|
||||
const atoms = getStateAtoms();
|
||||
for (const atom of atoms) {
|
||||
for (const e of (atom.edges || [])) {
|
||||
addPersonTerm(candidate, e?.s);
|
||||
addPersonTerm(candidate, e?.t);
|
||||
}
|
||||
}
|
||||
if (context?.name1) {
|
||||
candidate.delete(normalize(context.name1));
|
||||
}
|
||||
return candidate;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build character pools with trust tiers.
|
||||
* trustedCharacters: main/arcs/name2/L2 participants (clean source)
|
||||
* candidateCharacters: L0 edges.s/t (blacklist-cleaned)
|
||||
*/
|
||||
export function buildCharacterPools(store, context) {
|
||||
const trustedCharacters = collectTrustedCharacters(store, context);
|
||||
const candidateCharacters = collectCandidateCharactersFromL0(context);
|
||||
const allCharacters = new Set([...trustedCharacters, ...candidateCharacters]);
|
||||
return { trustedCharacters, candidateCharacters, allCharacters };
|
||||
}
|
||||
|
||||
/**
|
||||
* 构建实体词典
|
||||
*
|
||||
* 来源（按可信度）：
|
||||
* 1. store.json.characters.main — 已确认主要角色
|
||||
* 2. store.json.arcs[].name — 弧光对象
|
||||
* 3. context.name2 — 当前角色
|
||||
* 4. store.json.events[].participants — L2 事件参与者
|
||||
* 5. L0 atoms edges.s/edges.t
|
||||
*
|
||||
* 硬约束：永远排除 normalize(context.name1)
|
||||
*
|
||||
* @param {object} store - getSummaryStore() 返回值
|
||||
* @param {object} context - { name1: string, name2: string }
|
||||
* @returns {Set<string>} 标准化后的实体集合
|
||||
*/
|
||||
export function buildEntityLexicon(store, context) {
|
||||
return buildCharacterPools(store, context).allCharacters;
|
||||
}
|
||||
|
||||
/**
|
||||
* 构建"原词形 → 标准化"映射表
|
||||
* 用于从 lexicon 反查原始显示名
|
||||
*
|
||||
* @param {object} store
|
||||
* @param {object} context
|
||||
* @returns {Map<string, string>} normalize(name) → 原词形
|
||||
*/
|
||||
export function buildDisplayNameMap(store, context) {
|
||||
const map = new Map();
|
||||
|
||||
const register = (raw) => {
|
||||
const n = normalize(raw);
|
||||
if (!n || n.length < 2) return;
|
||||
if (isBlacklistedPersonTerm(n)) return;
|
||||
if (!map.has(n)) {
|
||||
map.set(n, String(raw).trim());
|
||||
}
|
||||
};
|
||||
|
||||
const main = store?.json?.characters?.main || [];
|
||||
for (const m of main) {
|
||||
register(typeof m === 'string' ? m : m.name);
|
||||
}
|
||||
|
||||
const arcs = store?.json?.arcs || [];
|
||||
for (const a of arcs) {
|
||||
register(a.name);
|
||||
}
|
||||
|
||||
if (context?.name2) register(context.name2);
|
||||
|
||||
// 4. L2 events 参与者
|
||||
const events = store?.json?.events || [];
|
||||
for (const ev of events) {
|
||||
for (const p of (ev?.participants || [])) {
|
||||
register(p);
|
||||
}
|
||||
}
|
||||
|
||||
// 5. L0 atoms 的 edges.s/edges.t
|
||||
const atoms = getStateAtoms();
|
||||
for (const atom of atoms) {
|
||||
for (const e of (atom.edges || [])) {
|
||||
register(e?.s);
|
||||
register(e?.t);
|
||||
}
|
||||
}
|
||||
|
||||
// ★ 硬约束：删除 name1
|
||||
if (context?.name1) {
|
||||
map.delete(normalize(context.name1));
|
||||
}
|
||||
|
||||
return map;
|
||||
}
|
||||
|
||||
/**
|
||||
* 从文本中提取命中的实体
|
||||
*
|
||||
* 逻辑：遍历词典，检查文本中是否包含（不区分大小写）
|
||||
* 返回命中的实体原词形（去重）
|
||||
*
|
||||
* @param {string} text - 清洗后的文本
|
||||
* @param {Set<string>} lexicon - 标准化后的实体集合
|
||||
* @param {Map<string, string>} displayMap - normalize → 原词形
|
||||
* @returns {string[]} 命中的实体（原词形）
|
||||
*/
|
||||
export function extractEntitiesFromText(text, lexicon, displayMap) {
|
||||
if (!text || !lexicon?.size) return [];
|
||||
|
||||
const textNorm = normalize(text);
|
||||
const hits = [];
|
||||
const seen = new Set();
|
||||
|
||||
for (const entity of lexicon) {
|
||||
if (textNorm.includes(entity) && !seen.has(entity)) {
|
||||
seen.add(entity);
|
||||
// 优先返回原词形
|
||||
const display = displayMap?.get(entity) || entity;
|
||||
hits.push(display);
|
||||
}
|
||||
}
|
||||
|
||||
return hits;
|
||||
}
|
||||
541
modules/story-summary/vector/retrieval/lexical-index.js
Normal file
541
modules/story-summary/vector/retrieval/lexical-index.js
Normal file
@@ -0,0 +1,541 @@
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
// lexical-index.js - MiniSearch 词法检索索引
|
||||
//
|
||||
// 职责：
|
||||
// 1. 对 L0 atoms + L1 chunks + L2 events 建立词法索引
|
||||
// 2. 提供词法检索接口（专名精确匹配兜底）
|
||||
// 3. 惰性构建 + 异步预热 + 缓存失效机制
|
||||
//
|
||||
// 索引存储：纯内存（不持久化）
|
||||
// 分词器：统一使用 tokenizer.js（结巴 + 实体保护 + 降级）
|
||||
// 重建时机：CHAT_CHANGED / L0提取完成 / L2总结完成
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
|
||||
import MiniSearch from '../../../../libs/minisearch.mjs';
|
||||
import { getContext } from '../../../../../../../extensions.js';
|
||||
import { getSummaryStore } from '../../data/store.js';
|
||||
import { getAllChunks } from '../storage/chunk-store.js';
|
||||
import { xbLog } from '../../../../core/debug-core.js';
|
||||
import { tokenizeForIndex } from '../utils/tokenizer.js';
|
||||
|
||||
const MODULE_ID = 'lexical-index';
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
// 缓存
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
|
||||
/** @type {MiniSearch|null} */
|
||||
let cachedIndex = null;
|
||||
|
||||
/** @type {string|null} */
|
||||
let cachedChatId = null;
|
||||
|
||||
/** @type {string|null} 数据指纹（atoms + chunks + events 数量） */
|
||||
let cachedFingerprint = null;
|
||||
|
||||
/** @type {boolean} 是否正在构建 */
|
||||
let building = false;
|
||||
|
||||
/** @type {Promise<MiniSearch|null>|null} 当前构建 Promise（防重入） */
|
||||
let buildPromise = null;
|
||||
/** @type {Map<number, string[]>} floor → 该楼层的 doc IDs（仅 L1 chunks） */
|
||||
let floorDocIds = new Map();
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
// 工具函数
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* 清理事件摘要（移除楼层标记）
|
||||
* @param {string} summary
|
||||
* @returns {string}
|
||||
*/
|
||||
function cleanSummary(summary) {
|
||||
return String(summary || '')
|
||||
.replace(/\s*\(#\d+(?:-\d+)?\)\s*$/, '')
|
||||
.trim();
|
||||
}
|
||||
|
||||
/**
|
||||
* 计算缓存指纹
|
||||
* @param {number} chunkCount
|
||||
* @param {number} eventCount
|
||||
* @returns {string}
|
||||
*/
|
||||
function computeFingerprint(chunkCount, eventCount) {
|
||||
return `${chunkCount}:${eventCount}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* 让出主线程（避免长时间阻塞 UI）
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
function yieldToMain() {
|
||||
return new Promise(resolve => setTimeout(resolve, 0));
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
// 文档收集
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* 收集所有待索引文档
|
||||
*
|
||||
* @param {object[]} chunks - getAllChunks(chatId) 返回值
|
||||
* @param {object[]} events - store.json.events
|
||||
* @returns {object[]} 文档数组
|
||||
*/
|
||||
function collectDocuments(chunks, events) {
|
||||
const docs = [];
|
||||
|
||||
// L1 chunks + 填充 floorDocIds
|
||||
for (const chunk of (chunks || [])) {
|
||||
if (!chunk?.chunkId || !chunk.text) continue;
|
||||
|
||||
const floor = chunk.floor ?? -1;
|
||||
docs.push({
|
||||
id: chunk.chunkId,
|
||||
type: 'chunk',
|
||||
floor,
|
||||
text: chunk.text,
|
||||
});
|
||||
|
||||
if (floor >= 0) {
|
||||
if (!floorDocIds.has(floor)) {
|
||||
floorDocIds.set(floor, []);
|
||||
}
|
||||
floorDocIds.get(floor).push(chunk.chunkId);
|
||||
}
|
||||
}
|
||||
|
||||
// L2 events
|
||||
for (const ev of (events || [])) {
|
||||
if (!ev?.id) continue;
|
||||
const parts = [];
|
||||
if (ev.title) parts.push(ev.title);
|
||||
if (ev.participants?.length) parts.push(ev.participants.join(' '));
|
||||
const summary = cleanSummary(ev.summary);
|
||||
if (summary) parts.push(summary);
|
||||
const text = parts.join(' ').trim();
|
||||
if (!text) continue;
|
||||
|
||||
docs.push({
|
||||
id: ev.id,
|
||||
type: 'event',
|
||||
floor: null,
|
||||
text,
|
||||
});
|
||||
}
|
||||
|
||||
return docs;
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
// 索引构建（分片，不阻塞主线程）
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
|
||||
/** 每批添加的文档数 */
|
||||
const BUILD_BATCH_SIZE = 500;
|
||||
|
||||
/**
|
||||
* 构建 MiniSearch 索引（分片异步）
|
||||
*
|
||||
* @param {object[]} docs - 文档数组
|
||||
* @returns {Promise<MiniSearch>}
|
||||
*/
|
||||
async function buildIndexAsync(docs) {
|
||||
const T0 = performance.now();
|
||||
|
||||
const index = new MiniSearch({
|
||||
fields: ['text'],
|
||||
storeFields: ['type', 'floor'],
|
||||
idField: 'id',
|
||||
searchOptions: {
|
||||
boost: { text: 1 },
|
||||
fuzzy: 0.2,
|
||||
prefix: true,
|
||||
},
|
||||
tokenize: tokenizeForIndex,
|
||||
});
|
||||
|
||||
if (!docs.length) {
|
||||
return index;
|
||||
}
|
||||
|
||||
// 分片添加，每批 BUILD_BATCH_SIZE 条后让出主线程
|
||||
for (let i = 0; i < docs.length; i += BUILD_BATCH_SIZE) {
|
||||
const batch = docs.slice(i, i + BUILD_BATCH_SIZE);
|
||||
index.addAll(batch);
|
||||
|
||||
// 非最后一批时让出主线程
|
||||
if (i + BUILD_BATCH_SIZE < docs.length) {
|
||||
await yieldToMain();
|
||||
}
|
||||
}
|
||||
|
||||
const elapsed = Math.round(performance.now() - T0);
|
||||
xbLog.info(MODULE_ID,
|
||||
`索引构建完成: ${docs.length} 文档 (${elapsed}ms)`
|
||||
);
|
||||
|
||||
return index;
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
// 检索
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* @typedef {object} LexicalSearchResult
|
||||
* @property {string[]} atomIds - 命中的 L0 atom IDs
|
||||
* @property {Set<number>} atomFloors - 命中的 L0 楼层集合
|
||||
* @property {string[]} chunkIds - 命中的 L1 chunk IDs
|
||||
* @property {Set<number>} chunkFloors - 命中的 L1 楼层集合
|
||||
* @property {string[]} eventIds - 命中的 L2 event IDs
|
||||
* @property {object[]} chunkScores - chunk 命中详情 [{ chunkId, score }]
|
||||
* @property {number} searchTime - 检索耗时 ms
|
||||
*/
|
||||
|
||||
/**
|
||||
* 在词法索引中检索
|
||||
*
|
||||
* @param {MiniSearch} index - 索引实例
|
||||
* @param {string[]} terms - 查询词列表
|
||||
* @returns {LexicalSearchResult}
|
||||
*/
|
||||
export function searchLexicalIndex(index, terms) {
|
||||
const T0 = performance.now();
|
||||
|
||||
const result = {
|
||||
atomIds: [],
|
||||
atomFloors: new Set(),
|
||||
chunkIds: [],
|
||||
chunkFloors: new Set(),
|
||||
eventIds: [],
|
||||
chunkScores: [],
|
||||
searchTime: 0,
|
||||
};
|
||||
|
||||
if (!index || !terms?.length) {
|
||||
result.searchTime = Math.round(performance.now() - T0);
|
||||
return result;
|
||||
}
|
||||
|
||||
// 用所有 terms 联合查询
|
||||
const queryString = terms.join(' ');
|
||||
|
||||
let hits;
|
||||
try {
|
||||
hits = index.search(queryString, {
|
||||
boost: { text: 1 },
|
||||
fuzzy: 0.2,
|
||||
prefix: true,
|
||||
combineWith: 'OR',
|
||||
// 使用与索引相同的分词器
|
||||
tokenize: tokenizeForIndex,
|
||||
});
|
||||
} catch (e) {
|
||||
xbLog.warn(MODULE_ID, '检索失败', e);
|
||||
result.searchTime = Math.round(performance.now() - T0);
|
||||
return result;
|
||||
}
|
||||
|
||||
// 分类结果
|
||||
const chunkIdSet = new Set();
|
||||
const eventIdSet = new Set();
|
||||
|
||||
for (const hit of hits) {
|
||||
const type = hit.type;
|
||||
const id = hit.id;
|
||||
const floor = hit.floor;
|
||||
|
||||
switch (type) {
|
||||
case 'chunk':
|
||||
if (!chunkIdSet.has(id)) {
|
||||
chunkIdSet.add(id);
|
||||
result.chunkIds.push(id);
|
||||
result.chunkScores.push({ chunkId: id, score: hit.score });
|
||||
if (typeof floor === 'number' && floor >= 0) {
|
||||
result.chunkFloors.add(floor);
|
||||
}
|
||||
}
|
||||
break;
|
||||
|
||||
case 'event':
|
||||
if (!eventIdSet.has(id)) {
|
||||
eventIdSet.add(id);
|
||||
result.eventIds.push(id);
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
result.searchTime = Math.round(performance.now() - T0);
|
||||
|
||||
xbLog.info(MODULE_ID,
|
||||
`检索完成: terms=[${terms.slice(0, 5).join(',')}] → atoms=${result.atomIds.length} chunks=${result.chunkIds.length} events=${result.eventIds.length} (${result.searchTime}ms)`
|
||||
);
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
// 内部构建流程（收集数据 + 构建索引）
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* 收集数据并构建索引
|
||||
*
|
||||
* @param {string} chatId
|
||||
* @returns {Promise<{index: MiniSearch, fingerprint: string}>}
|
||||
*/
|
||||
async function collectAndBuild(chatId) {
|
||||
// 清空侧索引（全量重建）
|
||||
floorDocIds = new Map();
|
||||
|
||||
// 收集数据（不含 L0 atoms）
|
||||
const store = getSummaryStore();
|
||||
const events = store?.json?.events || [];
|
||||
|
||||
let chunks = [];
|
||||
try {
|
||||
chunks = await getAllChunks(chatId);
|
||||
} catch (e) {
|
||||
xbLog.warn(MODULE_ID, '获取 chunks 失败', e);
|
||||
}
|
||||
|
||||
const fp = computeFingerprint(chunks.length, events.length);
|
||||
|
||||
// 检查是否在收集过程中缓存已被其他调用更新
|
||||
if (cachedIndex && cachedChatId === chatId && cachedFingerprint === fp) {
|
||||
return { index: cachedIndex, fingerprint: fp };
|
||||
}
|
||||
|
||||
// 收集文档（同时填充 floorDocIds）
|
||||
const docs = collectDocuments(chunks, events);
|
||||
|
||||
// 异步分片构建
|
||||
const index = await buildIndexAsync(docs);
|
||||
|
||||
return { index, fingerprint: fp };
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
// 公开接口：getLexicalIndex（惰性获取）
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* 获取词法索引（惰性构建 + 缓存）
|
||||
*
|
||||
* 如果缓存有效则直接返回；否则自动构建。
|
||||
* 如果正在构建中，等待构建完成。
|
||||
*
|
||||
* @returns {Promise<MiniSearch|null>}
|
||||
*/
|
||||
export async function getLexicalIndex() {
|
||||
const { chatId } = getContext();
|
||||
if (!chatId) return null;
|
||||
|
||||
// 快速路径：如果缓存存在且 chatId 未变，则直接命中
|
||||
// 指纹校验放到构建流程中完成，避免为指纹而额外读一次 IndexedDB
|
||||
if (cachedIndex && cachedChatId === chatId && cachedFingerprint) {
|
||||
return cachedIndex;
|
||||
}
|
||||
|
||||
// 正在构建中，等待结果
|
||||
if (building && buildPromise) {
|
||||
try {
|
||||
await buildPromise;
|
||||
if (cachedIndex && cachedChatId === chatId && cachedFingerprint) {
|
||||
return cachedIndex;
|
||||
}
|
||||
} catch {
|
||||
// 构建失败，继续往下重建
|
||||
}
|
||||
}
|
||||
|
||||
// 需要重建（指纹将在 collectAndBuild 内部计算并写入缓存）
|
||||
xbLog.info(MODULE_ID, `缓存失效，重建索引 (chatId=${chatId.slice(0, 8)})`);
|
||||
|
||||
building = true;
|
||||
buildPromise = collectAndBuild(chatId);
|
||||
|
||||
try {
|
||||
const { index, fingerprint } = await buildPromise;
|
||||
|
||||
// 原子替换缓存
|
||||
cachedIndex = index;
|
||||
cachedChatId = chatId;
|
||||
cachedFingerprint = fingerprint;
|
||||
|
||||
return index;
|
||||
} catch (e) {
|
||||
xbLog.error(MODULE_ID, '索引构建失败', e);
|
||||
return null;
|
||||
} finally {
|
||||
building = false;
|
||||
buildPromise = null;
|
||||
}
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
// 公开接口：warmupIndex（异步预建）
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* 异步预建索引
|
||||
*
|
||||
* 在 CHAT_CHANGED 时调用，后台构建索引。
|
||||
* 不阻塞调用方，不返回结果。
|
||||
* 构建完成后缓存自动更新，后续 getLexicalIndex() 直接命中。
|
||||
*
|
||||
* 调用时机：
|
||||
* - handleChatChanged（实体注入后）
|
||||
* - L0 提取完成
|
||||
* - L2 总结完成
|
||||
*/
|
||||
export function warmupIndex() {
|
||||
const { chatId } = getContext();
|
||||
if (!chatId) return;
|
||||
|
||||
// 已在构建中，不重复触发
|
||||
if (building) return;
|
||||
|
||||
// fire-and-forget
|
||||
getLexicalIndex().catch(e => {
|
||||
xbLog.warn(MODULE_ID, '预热索引失败', e);
|
||||
});
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
// 公开接口：invalidateLexicalIndex（缓存失效）
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* 使缓存失效（下次 getLexicalIndex / warmupIndex 时自动重建）
|
||||
*
|
||||
* 调用时机：
|
||||
* - CHAT_CHANGED
|
||||
* - L0 提取完成
|
||||
* - L2 总结完成
|
||||
*/
|
||||
export function invalidateLexicalIndex() {
|
||||
if (cachedIndex) {
|
||||
xbLog.info(MODULE_ID, '索引缓存已失效');
|
||||
}
|
||||
cachedIndex = null;
|
||||
cachedChatId = null;
|
||||
cachedFingerprint = null;
|
||||
floorDocIds = new Map();
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
// 增量更新接口
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* 为指定楼层添加 L1 chunks 到索引
|
||||
*
|
||||
* 先移除该楼层旧文档，再添加新文档。
|
||||
* 如果索引不存在（缓存失效），静默跳过（下次 getLexicalIndex 全量重建）。
|
||||
*
|
||||
* @param {number} floor - 楼层号
|
||||
* @param {object[]} chunks - chunk 对象列表（需有 chunkId、text、floor）
|
||||
*/
|
||||
export function addDocumentsForFloor(floor, chunks) {
|
||||
if (!cachedIndex || !chunks?.length) return;
|
||||
|
||||
// 先移除旧文档
|
||||
removeDocumentsByFloor(floor);
|
||||
|
||||
const docs = [];
|
||||
const docIds = [];
|
||||
|
||||
for (const chunk of chunks) {
|
||||
if (!chunk?.chunkId || !chunk.text) continue;
|
||||
docs.push({
|
||||
id: chunk.chunkId,
|
||||
type: 'chunk',
|
||||
floor: chunk.floor ?? floor,
|
||||
text: chunk.text,
|
||||
});
|
||||
docIds.push(chunk.chunkId);
|
||||
}
|
||||
|
||||
if (docs.length > 0) {
|
||||
cachedIndex.addAll(docs);
|
||||
floorDocIds.set(floor, docIds);
|
||||
xbLog.info(MODULE_ID, `增量添加: floor ${floor}, ${docs.length} 个 chunk`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 从索引中移除指定楼层的所有 L1 chunk 文档
|
||||
*
|
||||
* 使用 MiniSearch discard()（软删除）。
|
||||
* 如果索引不存在，静默跳过。
|
||||
*
|
||||
* @param {number} floor - 楼层号
|
||||
*/
|
||||
export function removeDocumentsByFloor(floor) {
|
||||
if (!cachedIndex) return;
|
||||
|
||||
const docIds = floorDocIds.get(floor);
|
||||
if (!docIds?.length) return;
|
||||
|
||||
for (const id of docIds) {
|
||||
try {
|
||||
cachedIndex.discard(id);
|
||||
} catch {
|
||||
// 文档可能不存在（已被全量重建替换）
|
||||
}
|
||||
}
|
||||
|
||||
floorDocIds.delete(floor);
|
||||
xbLog.info(MODULE_ID, `增量移除: floor ${floor}, ${docIds.length} 个文档`);
|
||||
}
|
||||
|
||||
/**
|
||||
* 将新 L2 事件添加到索引
|
||||
*
|
||||
* 如果事件 ID 已存在，先 discard 再 add（覆盖）。
|
||||
* 如果索引不存在，静默跳过。
|
||||
*
|
||||
* @param {object[]} events - 事件对象列表（需有 id、title、summary 等）
|
||||
*/
|
||||
export function addEventDocuments(events) {
|
||||
if (!cachedIndex || !events?.length) return;
|
||||
|
||||
const docs = [];
|
||||
|
||||
for (const ev of events) {
|
||||
if (!ev?.id) continue;
|
||||
|
||||
const parts = [];
|
||||
if (ev.title) parts.push(ev.title);
|
||||
if (ev.participants?.length) parts.push(ev.participants.join(' '));
|
||||
const summary = cleanSummary(ev.summary);
|
||||
if (summary) parts.push(summary);
|
||||
const text = parts.join(' ').trim();
|
||||
if (!text) continue;
|
||||
|
||||
// 覆盖：先尝试移除旧的
|
||||
try {
|
||||
cachedIndex.discard(ev.id);
|
||||
} catch {
|
||||
// 不存在则忽略
|
||||
}
|
||||
|
||||
docs.push({
|
||||
id: ev.id,
|
||||
type: 'event',
|
||||
floor: null,
|
||||
text,
|
||||
});
|
||||
}
|
||||
|
||||
if (docs.length > 0) {
|
||||
cachedIndex.addAll(docs);
|
||||
xbLog.info(MODULE_ID, `增量添加: ${docs.length} 个事件`);
|
||||
}
|
||||
}
|
||||
685
modules/story-summary/vector/retrieval/metrics.js
Normal file
685
modules/story-summary/vector/retrieval/metrics.js
Normal file
@@ -0,0 +1,685 @@
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
// Story Summary - Metrics Collector (v6 - Dense-Gated Lexical)
|
||||
//
|
||||
// v5 → v6 变更：
|
||||
// - lexical: 新增 eventFilteredByDense / floorFilteredByDense
|
||||
// - event: entityFilter bypass 阈值改为 CONFIG 驱动（0.80）
|
||||
// - 其余结构不变
|
||||
//
|
||||
// v4 → v5 变更：
|
||||
// - query: 新增 segmentWeights / r2Weights（加权向量诊断）
|
||||
// - fusion: 新增 denseAggMethod / lexDensityBonus（聚合策略可观测）
|
||||
// - quality: 新增 rerankRetentionRate（粗排-精排一致性）
|
||||
// - 移除 timing 中从未写入的死字段（queryBuild/queryRefine/lexicalSearch/fusion）
|
||||
// - 移除从未写入的 arc 区块
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
|
||||
/**
|
||||
* 创建空的指标对象
|
||||
* @returns {object}
|
||||
*/
|
||||
export function createMetrics() {
|
||||
return {
|
||||
// Query Build - 查询构建
|
||||
query: {
|
||||
buildTime: 0,
|
||||
refineTime: 0,
|
||||
lengths: {
|
||||
v0Chars: 0,
|
||||
v1Chars: null, // null = 无 hints
|
||||
rerankChars: 0,
|
||||
},
|
||||
segmentWeights: [], // R1 归一化后权重 [context..., focus]
|
||||
r2Weights: null, // R2 归一化后权重 [context..., focus, hints]（null = 无 hints）
|
||||
},
|
||||
|
||||
// Anchor (L0 StateAtoms) - 语义锚点
|
||||
anchor: {
|
||||
needRecall: false,
|
||||
focusTerms: [],
|
||||
focusCharacters: [],
|
||||
focusEntities: [],
|
||||
matched: 0,
|
||||
floorsHit: 0,
|
||||
topHits: [],
|
||||
},
|
||||
|
||||
// Lexical (MiniSearch) - 词法检索
|
||||
lexical: {
|
||||
terms: [],
|
||||
atomHits: 0,
|
||||
chunkHits: 0,
|
||||
eventHits: 0,
|
||||
searchTime: 0,
|
||||
indexReadyTime: 0,
|
||||
eventFilteredByDense: 0,
|
||||
floorFilteredByDense: 0,
|
||||
},
|
||||
|
||||
// Fusion (W-RRF, floor-level) - 多路融合
|
||||
fusion: {
|
||||
denseFloors: 0,
|
||||
lexFloors: 0,
|
||||
totalUnique: 0,
|
||||
afterCap: 0,
|
||||
time: 0,
|
||||
denseAggMethod: '', // 聚合方法描述（如 "max×0.6+mean×0.4"）
|
||||
lexDensityBonus: 0, // 密度加成系数
|
||||
},
|
||||
|
||||
// Constraint (L3 Facts) - 世界约束
|
||||
constraint: {
|
||||
total: 0,
|
||||
filtered: 0,
|
||||
injected: 0,
|
||||
tokens: 0,
|
||||
samples: [],
|
||||
},
|
||||
|
||||
// Event (L2 Events) - 事件摘要
|
||||
event: {
|
||||
inStore: 0,
|
||||
considered: 0,
|
||||
selected: 0,
|
||||
byRecallType: { direct: 0, related: 0, causal: 0, lexical: 0, l0Linked: 0 },
|
||||
similarityDistribution: { min: 0, max: 0, mean: 0, median: 0 },
|
||||
entityFilter: null,
|
||||
causalChainDepth: 0,
|
||||
causalCount: 0,
|
||||
entitiesUsed: 0,
|
||||
focusTermsCount: 0,
|
||||
entityNames: [],
|
||||
},
|
||||
|
||||
// Evidence (Two-Stage: Floor rerank → L1 pull) - 原文证据
|
||||
evidence: {
|
||||
// Stage 1: Floor
|
||||
floorCandidates: 0,
|
||||
floorsSelected: 0,
|
||||
l0Collected: 0,
|
||||
rerankApplied: false,
|
||||
rerankFailed: false,
|
||||
beforeRerank: 0,
|
||||
afterRerank: 0,
|
||||
rerankTime: 0,
|
||||
rerankScores: null,
|
||||
rerankDocAvgLength: 0,
|
||||
|
||||
// Stage 2: L1
|
||||
l1Pulled: 0,
|
||||
l1Attached: 0,
|
||||
l1CosineTime: 0,
|
||||
|
||||
// 装配
|
||||
contextPairsAdded: 0,
|
||||
tokens: 0,
|
||||
assemblyTime: 0,
|
||||
},
|
||||
|
||||
// Diffusion (PPR Spreading Activation) - 图扩散
|
||||
diffusion: {
|
||||
seedCount: 0,
|
||||
graphNodes: 0,
|
||||
graphEdges: 0,
|
||||
candidatePairs: 0,
|
||||
pairsFromWhat: 0,
|
||||
pairsFromRSem: 0,
|
||||
rSemAvgSim: 0,
|
||||
timeWindowFilteredPairs: 0,
|
||||
topKPrunedPairs: 0,
|
||||
edgeDensity: 0,
|
||||
reweightWhoUsed: 0,
|
||||
reweightWhereUsed: 0,
|
||||
iterations: 0,
|
||||
convergenceError: 0,
|
||||
pprActivated: 0,
|
||||
cosineGatePassed: 0,
|
||||
cosineGateFiltered: 0,
|
||||
cosineGateNoVector: 0,
|
||||
postGatePassRate: 0,
|
||||
finalCount: 0,
|
||||
scoreDistribution: { min: 0, max: 0, mean: 0 },
|
||||
byChannel: { what: 0, where: 0, rSem: 0, who: 0 },
|
||||
time: 0,
|
||||
},
|
||||
|
||||
// Formatting - 格式化
|
||||
formatting: {
|
||||
sectionsIncluded: [],
|
||||
time: 0,
|
||||
},
|
||||
|
||||
// Budget Summary - 预算
|
||||
budget: {
|
||||
total: 0,
|
||||
limit: 0,
|
||||
utilization: 0,
|
||||
breakdown: {
|
||||
constraints: 0,
|
||||
events: 0,
|
||||
distantEvidence: 0,
|
||||
recentEvidence: 0,
|
||||
arcs: 0,
|
||||
},
|
||||
},
|
||||
|
||||
// Timing - 计时（仅包含实际写入的字段）
|
||||
timing: {
|
||||
anchorSearch: 0,
|
||||
constraintFilter: 0,
|
||||
eventRetrieval: 0,
|
||||
evidenceRetrieval: 0,
|
||||
evidenceRerank: 0,
|
||||
evidenceAssembly: 0,
|
||||
diffusion: 0,
|
||||
formatting: 0,
|
||||
total: 0,
|
||||
},
|
||||
|
||||
// Quality Indicators - 质量指标
|
||||
quality: {
|
||||
constraintCoverage: 100,
|
||||
eventPrecisionProxy: 0,
|
||||
l1AttachRate: 0,
|
||||
rerankRetentionRate: 0,
|
||||
diffusionEffectiveRate: 0,
|
||||
potentialIssues: [],
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* 计算相似度分布统计
|
||||
* @param {number[]} similarities
|
||||
* @returns {{min: number, max: number, mean: number, median: number}}
|
||||
*/
|
||||
export function calcSimilarityStats(similarities) {
|
||||
if (!similarities?.length) {
|
||||
return { min: 0, max: 0, mean: 0, median: 0 };
|
||||
}
|
||||
|
||||
const sorted = [...similarities].sort((a, b) => a - b);
|
||||
const sum = sorted.reduce((a, b) => a + b, 0);
|
||||
|
||||
return {
|
||||
min: Number(sorted[0].toFixed(3)),
|
||||
max: Number(sorted[sorted.length - 1].toFixed(3)),
|
||||
mean: Number((sum / sorted.length).toFixed(3)),
|
||||
median: Number(sorted[Math.floor(sorted.length / 2)].toFixed(3)),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* 格式化权重数组为紧凑字符串
|
||||
* @param {number[]|null} weights
|
||||
* @returns {string}
|
||||
*/
|
||||
function fmtWeights(weights) {
|
||||
if (!weights?.length) return 'N/A';
|
||||
return '[' + weights.map(w => (typeof w === 'number' ? w.toFixed(3) : String(w))).join(', ') + ']';
|
||||
}
|
||||
|
||||
/**
|
||||
* 格式化指标为可读日志
|
||||
* @param {object} metrics
|
||||
* @returns {string}
|
||||
*/
|
||||
export function formatMetricsLog(metrics) {
|
||||
const m = metrics;
|
||||
const lines = [];
|
||||
|
||||
lines.push('');
|
||||
lines.push('════════════════════════════════════════');
|
||||
lines.push(' Recall Metrics Report (v5) ');
|
||||
lines.push('════════════════════════════════════════');
|
||||
lines.push('');
|
||||
|
||||
// Query Length
|
||||
lines.push('[Query Length] 查询长度');
|
||||
lines.push(`├─ query_v0_chars: ${m.query?.lengths?.v0Chars ?? 0}`);
|
||||
lines.push(`├─ query_v1_chars: ${m.query?.lengths?.v1Chars == null ? 'N/A' : m.query.lengths.v1Chars}`);
|
||||
lines.push(`└─ rerank_query_chars: ${m.query?.lengths?.rerankChars ?? 0}`);
|
||||
lines.push('');
|
||||
|
||||
// Query Build
|
||||
lines.push('[Query] 查询构建');
|
||||
lines.push(`├─ build_time: ${m.query.buildTime}ms`);
|
||||
lines.push(`├─ refine_time: ${m.query.refineTime}ms`);
|
||||
lines.push(`├─ r1_weights: ${fmtWeights(m.query.segmentWeights)}`);
|
||||
if (m.query.r2Weights) {
|
||||
lines.push(`└─ r2_weights: ${fmtWeights(m.query.r2Weights)}`);
|
||||
} else {
|
||||
lines.push(`└─ r2_weights: N/A (no hints)`);
|
||||
}
|
||||
lines.push('');
|
||||
|
||||
// Anchor (L0 StateAtoms)
|
||||
lines.push('[Anchor] L0 StateAtoms - 语义锚点');
|
||||
lines.push(`├─ need_recall: ${m.anchor.needRecall}`);
|
||||
if (m.anchor.needRecall) {
|
||||
lines.push(`├─ focus_terms: [${(m.anchor.focusTerms || m.anchor.focusEntities || []).join(', ')}]`);
|
||||
lines.push(`├─ focus_characters: [${(m.anchor.focusCharacters || []).join(', ')}]`);
|
||||
lines.push(`├─ matched: ${m.anchor.matched || 0}`);
|
||||
lines.push(`└─ floors_hit: ${m.anchor.floorsHit || 0}`);
|
||||
}
|
||||
lines.push('');
|
||||
|
||||
// Lexical (MiniSearch)
|
||||
lines.push('[Lexical] MiniSearch - 词法检索');
|
||||
lines.push(`├─ terms: [${(m.lexical.terms || []).slice(0, 8).join(', ')}]`);
|
||||
lines.push(`├─ atom_hits: ${m.lexical.atomHits}`);
|
||||
lines.push(`├─ chunk_hits: ${m.lexical.chunkHits}`);
|
||||
lines.push(`├─ event_hits: ${m.lexical.eventHits}`);
|
||||
lines.push(`├─ search_time: ${m.lexical.searchTime}ms`);
|
||||
if (m.lexical.indexReadyTime > 0) {
|
||||
lines.push(`├─ index_ready_time: ${m.lexical.indexReadyTime}ms`);
|
||||
}
|
||||
if (m.lexical.eventFilteredByDense > 0) {
|
||||
lines.push(`├─ event_filtered_by_dense: ${m.lexical.eventFilteredByDense}`);
|
||||
}
|
||||
if (m.lexical.floorFilteredByDense > 0) {
|
||||
lines.push(`├─ floor_filtered_by_dense: ${m.lexical.floorFilteredByDense}`);
|
||||
}
|
||||
lines.push(`└─ dense_gate_threshold: 0.50`);
|
||||
lines.push('');
|
||||
|
||||
// Fusion (W-RRF, floor-level)
|
||||
lines.push('[Fusion] W-RRF (floor-level) - 多路融合');
|
||||
lines.push(`├─ dense_floors: ${m.fusion.denseFloors}`);
|
||||
lines.push(`├─ lex_floors: ${m.fusion.lexFloors}`);
|
||||
if (m.fusion.lexDensityBonus > 0) {
|
||||
lines.push(`│ └─ density_bonus: ${m.fusion.lexDensityBonus}`);
|
||||
}
|
||||
lines.push(`├─ total_unique: ${m.fusion.totalUnique}`);
|
||||
lines.push(`├─ after_cap: ${m.fusion.afterCap}`);
|
||||
lines.push(`└─ time: ${m.fusion.time}ms`);
|
||||
lines.push('');
|
||||
|
||||
// Constraint (L3 Facts)
|
||||
lines.push('[Constraint] L3 Facts - 世界约束');
|
||||
lines.push(`├─ total: ${m.constraint.total}`);
|
||||
lines.push(`├─ filtered: ${m.constraint.filtered || 0}`);
|
||||
lines.push(`├─ injected: ${m.constraint.injected}`);
|
||||
lines.push(`├─ tokens: ${m.constraint.tokens}`);
|
||||
if (m.constraint.samples && m.constraint.samples.length > 0) {
|
||||
lines.push(`└─ samples: "${m.constraint.samples.slice(0, 2).join('", "')}"`);
|
||||
}
|
||||
lines.push('');
|
||||
|
||||
// Event (L2 Events)
|
||||
lines.push('[Event] L2 Events - 事件摘要');
|
||||
lines.push(`├─ in_store: ${m.event.inStore}`);
|
||||
lines.push(`├─ considered: ${m.event.considered}`);
|
||||
|
||||
if (m.event.entityFilter) {
|
||||
const ef = m.event.entityFilter;
|
||||
lines.push(`├─ entity_filter:`);
|
||||
lines.push(`│ ├─ focus_characters: [${(ef.focusCharacters || ef.focusEntities || []).join(', ')}]`);
|
||||
lines.push(`│ ├─ before: ${ef.before}`);
|
||||
lines.push(`│ ├─ after: ${ef.after}`);
|
||||
lines.push(`│ └─ filtered: ${ef.filtered}`);
|
||||
}
|
||||
|
||||
lines.push(`├─ selected: ${m.event.selected}`);
|
||||
lines.push(`├─ by_recall_type:`);
|
||||
lines.push(`│ ├─ direct: ${m.event.byRecallType.direct}`);
|
||||
lines.push(`│ ├─ related: ${m.event.byRecallType.related}`);
|
||||
lines.push(`│ ├─ causal: ${m.event.byRecallType.causal}`);
|
||||
if (m.event.byRecallType.l0Linked) {
|
||||
lines.push(`│ ├─ lexical: ${m.event.byRecallType.lexical}`);
|
||||
lines.push(`│ └─ l0_linked: ${m.event.byRecallType.l0Linked}`);
|
||||
} else {
|
||||
lines.push(`│ └─ lexical: ${m.event.byRecallType.lexical}`);
|
||||
}
|
||||
|
||||
const sim = m.event.similarityDistribution;
|
||||
if (sim && sim.max > 0) {
|
||||
lines.push(`├─ similarity_distribution:`);
|
||||
lines.push(`│ ├─ min: ${sim.min}`);
|
||||
lines.push(`│ ├─ max: ${sim.max}`);
|
||||
lines.push(`│ ├─ mean: ${sim.mean}`);
|
||||
lines.push(`│ └─ median: ${sim.median}`);
|
||||
}
|
||||
|
||||
lines.push(`├─ causal_chain: depth=${m.event.causalChainDepth}, count=${m.event.causalCount}`);
|
||||
lines.push(`└─ focus_characters_used: ${m.event.entitiesUsed} [${(m.event.entityNames || []).join(', ')}], focus_terms_count=${m.event.focusTermsCount || 0}`);
|
||||
lines.push('');
|
||||
|
||||
// Evidence (Two-Stage: Floor Rerank → L1 Pull)
|
||||
lines.push('[Evidence] Two-Stage: Floor Rerank → L1 Pull');
|
||||
lines.push(`├─ Stage 1 (Floor Rerank):`);
|
||||
lines.push(`│ ├─ floor_candidates (post-fusion): ${m.evidence.floorCandidates}`);
|
||||
|
||||
if (m.evidence.rerankApplied) {
|
||||
lines.push(`│ ├─ rerank_applied: true`);
|
||||
if (m.evidence.rerankFailed) {
|
||||
lines.push(`│ │ ⚠ rerank_failed: using fusion order`);
|
||||
}
|
||||
lines.push(`│ │ ├─ before: ${m.evidence.beforeRerank} floors`);
|
||||
lines.push(`│ │ ├─ after: ${m.evidence.afterRerank} floors`);
|
||||
lines.push(`│ │ └─ time: ${m.evidence.rerankTime}ms`);
|
||||
if (m.evidence.rerankScores) {
|
||||
const rs = m.evidence.rerankScores;
|
||||
lines.push(`│ ├─ rerank_scores: min=${rs.min}, max=${rs.max}, mean=${rs.mean}`);
|
||||
}
|
||||
if (m.evidence.rerankDocAvgLength > 0) {
|
||||
lines.push(`│ ├─ rerank_doc_avg_length: ${m.evidence.rerankDocAvgLength} chars`);
|
||||
}
|
||||
} else {
|
||||
lines.push(`│ ├─ rerank_applied: false`);
|
||||
}
|
||||
|
||||
lines.push(`│ ├─ floors_selected: ${m.evidence.floorsSelected}`);
|
||||
lines.push(`│ └─ l0_atoms_collected: ${m.evidence.l0Collected}`);
|
||||
lines.push(`├─ Stage 2 (L1):`);
|
||||
lines.push(`│ ├─ pulled: ${m.evidence.l1Pulled}`);
|
||||
lines.push(`│ ├─ attached: ${m.evidence.l1Attached}`);
|
||||
lines.push(`│ └─ cosine_time: ${m.evidence.l1CosineTime}ms`);
|
||||
lines.push(`├─ tokens: ${m.evidence.tokens}`);
|
||||
lines.push(`└─ assembly_time: ${m.evidence.assemblyTime}ms`);
|
||||
lines.push('');
|
||||
|
||||
// Diffusion (PPR)
|
||||
lines.push('[Diffusion] PPR Spreading Activation');
|
||||
lines.push(`├─ seeds: ${m.diffusion.seedCount}`);
|
||||
lines.push(`├─ graph: ${m.diffusion.graphNodes} nodes, ${m.diffusion.graphEdges} edges`);
|
||||
lines.push(`├─ candidate_pairs: ${m.diffusion.candidatePairs || 0} (what=${m.diffusion.pairsFromWhat || 0}, r_sem=${m.diffusion.pairsFromRSem || 0})`);
|
||||
lines.push(`├─ r_sem_avg_sim: ${m.diffusion.rSemAvgSim || 0}`);
|
||||
lines.push(`├─ pair_filters: time_window=${m.diffusion.timeWindowFilteredPairs || 0}, topk_pruned=${m.diffusion.topKPrunedPairs || 0}`);
|
||||
lines.push(`├─ edge_density: ${m.diffusion.edgeDensity || 0}%`);
|
||||
if (m.diffusion.graphEdges > 0) {
|
||||
const ch = m.diffusion.byChannel || {};
|
||||
lines.push(`│ ├─ by_channel: what=${ch.what || 0}, r_sem=${ch.rSem || 0}, who=${ch.who || 0}, where=${ch.where || 0}`);
|
||||
lines.push(`│ └─ reweight_used: who=${m.diffusion.reweightWhoUsed || 0}, where=${m.diffusion.reweightWhereUsed || 0}`);
|
||||
}
|
||||
if (m.diffusion.iterations > 0) {
|
||||
lines.push(`├─ ppr: ${m.diffusion.iterations} iterations, ε=${Number(m.diffusion.convergenceError).toExponential(1)}`);
|
||||
}
|
||||
lines.push(`├─ activated (excl seeds): ${m.diffusion.pprActivated}`);
|
||||
if (m.diffusion.pprActivated > 0) {
|
||||
lines.push(`├─ cosine_gate: ${m.diffusion.cosineGatePassed} passed, ${m.diffusion.cosineGateFiltered} filtered`);
|
||||
const passPrefix = m.diffusion.cosineGateNoVector > 0 ? '│ ├─' : '│ └─';
|
||||
lines.push(`${passPrefix} pass_rate: ${m.diffusion.postGatePassRate || 0}%`);
|
||||
if (m.diffusion.cosineGateNoVector > 0) {
|
||||
lines.push(`│ ├─ no_vector: ${m.diffusion.cosineGateNoVector}`);
|
||||
}
|
||||
}
|
||||
lines.push(`├─ final_injected: ${m.diffusion.finalCount}`);
|
||||
if (m.diffusion.finalCount > 0) {
|
||||
const ds = m.diffusion.scoreDistribution;
|
||||
lines.push(`├─ scores: min=${ds.min}, max=${ds.max}, mean=${ds.mean}`);
|
||||
}
|
||||
lines.push(`└─ time: ${m.diffusion.time}ms`);
|
||||
lines.push('');
|
||||
|
||||
// Formatting
|
||||
lines.push('[Formatting] 格式化');
|
||||
lines.push(`├─ sections: [${(m.formatting.sectionsIncluded || []).join(', ')}]`);
|
||||
lines.push(`└─ time: ${m.formatting.time}ms`);
|
||||
lines.push('');
|
||||
|
||||
// Budget Summary
|
||||
lines.push('[Budget] 预算');
|
||||
lines.push(`├─ total_tokens: ${m.budget.total}`);
|
||||
lines.push(`├─ limit: ${m.budget.limit}`);
|
||||
lines.push(`├─ utilization: ${m.budget.utilization}%`);
|
||||
lines.push(`└─ breakdown:`);
|
||||
const bd = m.budget.breakdown || {};
|
||||
lines.push(` ├─ constraints: ${bd.constraints || 0}`);
|
||||
lines.push(` ├─ events: ${bd.events || 0}`);
|
||||
lines.push(` ├─ distant_evidence: ${bd.distantEvidence || 0}`);
|
||||
lines.push(` ├─ recent_evidence: ${bd.recentEvidence || 0}`);
|
||||
lines.push(` └─ arcs: ${bd.arcs || 0}`);
|
||||
lines.push('');
|
||||
|
||||
// Timing
|
||||
lines.push('[Timing] 计时');
|
||||
lines.push(`├─ query_build: ${m.query.buildTime}ms`);
|
||||
lines.push(`├─ query_refine: ${m.query.refineTime}ms`);
|
||||
lines.push(`├─ anchor_search: ${m.timing.anchorSearch}ms`);
|
||||
const lexicalTotal = (m.lexical.searchTime || 0) + (m.lexical.indexReadyTime || 0);
|
||||
lines.push(`├─ lexical_search: ${lexicalTotal}ms (query=${m.lexical.searchTime || 0}ms, index_ready=${m.lexical.indexReadyTime || 0}ms)`);
|
||||
lines.push(`├─ fusion: ${m.fusion.time}ms`);
|
||||
lines.push(`├─ constraint_filter: ${m.timing.constraintFilter}ms`);
|
||||
lines.push(`├─ event_retrieval: ${m.timing.eventRetrieval}ms`);
|
||||
lines.push(`├─ evidence_retrieval: ${m.timing.evidenceRetrieval}ms`);
|
||||
lines.push(`├─ floor_rerank: ${m.timing.evidenceRerank || 0}ms`);
|
||||
lines.push(`├─ l1_cosine: ${m.evidence.l1CosineTime}ms`);
|
||||
lines.push(`├─ diffusion: ${m.timing.diffusion}ms`);
|
||||
lines.push(`├─ evidence_assembly: ${m.timing.evidenceAssembly}ms`);
|
||||
lines.push(`├─ formatting: ${m.timing.formatting}ms`);
|
||||
lines.push(`└─ total: ${m.timing.total}ms`);
|
||||
lines.push('');
|
||||
|
||||
// Quality Indicators
|
||||
lines.push('[Quality] 质量指标');
|
||||
lines.push(`├─ constraint_coverage: ${m.quality.constraintCoverage}%`);
|
||||
lines.push(`├─ event_precision_proxy: ${m.quality.eventPrecisionProxy}`);
|
||||
lines.push(`├─ l1_attach_rate: ${m.quality.l1AttachRate}%`);
|
||||
lines.push(`├─ rerank_retention_rate: ${m.quality.rerankRetentionRate}%`);
|
||||
lines.push(`├─ diffusion_effective_rate: ${m.quality.diffusionEffectiveRate}%`);
|
||||
|
||||
if (m.quality.potentialIssues && m.quality.potentialIssues.length > 0) {
|
||||
lines.push(`└─ potential_issues:`);
|
||||
m.quality.potentialIssues.forEach((issue, i) => {
|
||||
const prefix = i === m.quality.potentialIssues.length - 1 ? ' └─' : ' ├─';
|
||||
lines.push(`${prefix} ⚠ ${issue}`);
|
||||
});
|
||||
} else {
|
||||
lines.push(`└─ potential_issues: none`);
|
||||
}
|
||||
|
||||
lines.push('');
|
||||
lines.push('════════════════════════════════════════');
|
||||
lines.push('');
|
||||
|
||||
return lines.join('\n');
|
||||
}
|
||||
|
||||
/**
|
||||
* 检测潜在问题
|
||||
* @param {object} metrics
|
||||
* @returns {string[]}
|
||||
*/
|
||||
export function detectIssues(metrics) {
|
||||
const issues = [];
|
||||
const m = metrics;
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────
|
||||
// 查询构建问题
|
||||
// ─────────────────────────────────────────────────────────────────
|
||||
|
||||
if ((m.anchor.focusTerms || m.anchor.focusEntities || []).length === 0) {
|
||||
issues.push('No focus entities extracted - entity lexicon may be empty or messages too short');
|
||||
}
|
||||
|
||||
// 权重极端退化检测
|
||||
const segWeights = m.query.segmentWeights || [];
|
||||
if (segWeights.length > 0) {
|
||||
const focusWeight = segWeights[segWeights.length - 1] || 0;
|
||||
if (focusWeight < 0.15) {
|
||||
issues.push(`Focus segment weight very low (${(focusWeight * 100).toFixed(0)}%) - focus message may be too short`);
|
||||
}
|
||||
const allLow = segWeights.every(w => w < 0.1);
|
||||
if (allLow) {
|
||||
issues.push('All segment weights below 10% - all messages may be extremely short');
|
||||
}
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────
|
||||
// 锚点匹配问题
|
||||
// ─────────────────────────────────────────────────────────────────
|
||||
|
||||
if ((m.anchor.matched || 0) === 0 && m.anchor.needRecall) {
|
||||
issues.push('No anchors matched - may need to generate anchors');
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────
|
||||
// 词法检索问题
|
||||
// ─────────────────────────────────────────────────────────────────
|
||||
|
||||
if ((m.lexical.terms || []).length > 0 && m.lexical.chunkHits === 0 && m.lexical.eventHits === 0) {
|
||||
issues.push('Lexical search returned zero hits - terms may not match any indexed content');
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────
|
||||
// 融合问题（floor-level）
|
||||
// ─────────────────────────────────────────────────────────────────
|
||||
|
||||
if (m.fusion.lexFloors === 0 && m.fusion.denseFloors > 0) {
|
||||
issues.push('No lexical floors in fusion - hybrid retrieval not contributing');
|
||||
}
|
||||
|
||||
if (m.fusion.afterCap === 0) {
|
||||
issues.push('Fusion produced zero floor candidates - all retrieval paths may have failed');
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────
|
||||
// 事件召回问题
|
||||
// ─────────────────────────────────────────────────────────────────
|
||||
|
||||
if (m.event.considered > 0) {
|
||||
const denseSelected =
|
||||
(m.event.byRecallType?.direct || 0) +
|
||||
(m.event.byRecallType?.related || 0);
|
||||
|
||||
const denseSelectRatio = denseSelected / m.event.considered;
|
||||
|
||||
if (denseSelectRatio < 0.1) {
|
||||
issues.push(`Dense event selection ratio too low (${(denseSelectRatio * 100).toFixed(1)}%) - threshold may be too high`);
|
||||
}
|
||||
if (denseSelectRatio > 0.6 && m.event.considered > 10) {
|
||||
issues.push(`Dense event selection ratio high (${(denseSelectRatio * 100).toFixed(1)}%) - may include noise`);
|
||||
}
|
||||
}
|
||||
|
||||
// 实体过滤问题
|
||||
if (m.event.entityFilter) {
|
||||
const ef = m.event.entityFilter;
|
||||
if (ef.filtered === 0 && ef.before > 10) {
|
||||
issues.push('No events filtered by entity - focus entities may be too broad or missing');
|
||||
}
|
||||
if (ef.before > 0 && ef.filtered > ef.before * 0.8) {
|
||||
issues.push(`Too many events filtered (${ef.filtered}/${ef.before}) - focus may be too narrow`);
|
||||
}
|
||||
}
|
||||
|
||||
// 相似度问题
|
||||
if (m.event.similarityDistribution && m.event.similarityDistribution.min > 0 && m.event.similarityDistribution.min < 0.5) {
|
||||
issues.push(`Low similarity events included (min=${m.event.similarityDistribution.min})`);
|
||||
}
|
||||
|
||||
// 因果链问题
|
||||
if (m.event.selected > 0 && m.event.causalCount === 0 && m.event.byRecallType.direct === 0) {
|
||||
issues.push('No direct or causal events - query may not align with stored events');
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────
|
||||
// Floor Rerank 问题
|
||||
// ─────────────────────────────────────────────────────────────────
|
||||
|
||||
if (m.evidence.rerankFailed) {
|
||||
issues.push('Rerank API failed — using fusion rank order as fallback, relevance scores are zero');
|
||||
}
|
||||
|
||||
if (m.evidence.rerankApplied && !m.evidence.rerankFailed) {
|
||||
if (m.evidence.rerankScores) {
|
||||
const rs = m.evidence.rerankScores;
|
||||
if (rs.max < 0.3) {
|
||||
issues.push(`Low floor rerank scores (max=${rs.max}) - query-document domain mismatch`);
|
||||
}
|
||||
if (rs.mean < 0.2) {
|
||||
issues.push(`Very low average floor rerank score (mean=${rs.mean}) - context may be weak`);
|
||||
}
|
||||
}
|
||||
|
||||
if (m.evidence.rerankTime > 3000) {
|
||||
issues.push(`Slow floor rerank (${m.evidence.rerankTime}ms) - may affect response time`);
|
||||
}
|
||||
|
||||
if (m.evidence.rerankDocAvgLength > 3000) {
|
||||
issues.push(`Large rerank documents (avg ${m.evidence.rerankDocAvgLength} chars) - may reduce rerank precision`);
|
||||
}
|
||||
}
|
||||
|
||||
// Rerank 保留率
|
||||
const retentionRate = m.evidence.floorCandidates > 0
|
||||
? Math.round(m.evidence.floorsSelected / m.evidence.floorCandidates * 100)
|
||||
: 0;
|
||||
m.quality.rerankRetentionRate = retentionRate;
|
||||
|
||||
if (m.evidence.floorCandidates > 0 && retentionRate < 25) {
|
||||
issues.push(`Low rerank retention rate (${retentionRate}%) - fusion ranking poorly aligned with reranker`);
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────
|
||||
// L1 挂载问题
|
||||
// ─────────────────────────────────────────────────────────────────
|
||||
|
||||
if (m.evidence.floorsSelected > 0 && m.evidence.l1Pulled === 0) {
|
||||
issues.push('Zero L1 chunks pulled - L1 vectors may not exist or DB read failed');
|
||||
}
|
||||
|
||||
if (m.evidence.floorsSelected > 0 && m.evidence.l1Attached === 0 && m.evidence.l1Pulled > 0) {
|
||||
issues.push('L1 chunks pulled but none attached - cosine scores may be too low');
|
||||
}
|
||||
|
||||
const l1AttachRate = m.quality.l1AttachRate || 0;
|
||||
if (m.evidence.floorsSelected > 3 && l1AttachRate < 50) {
|
||||
issues.push(`Low L1 attach rate (${l1AttachRate}%) - selected floors lack L1 chunks`);
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────
|
||||
// 预算问题
|
||||
// ─────────────────────────────────────────────────────────────────
|
||||
|
||||
if (m.budget.utilization > 90) {
|
||||
issues.push(`High budget utilization (${m.budget.utilization}%) - may be truncating content`);
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────
|
||||
// 性能问题
|
||||
// ─────────────────────────────────────────────────────────────────
|
||||
|
||||
if (m.timing.total > 8000) {
|
||||
issues.push(`Slow recall (${m.timing.total}ms) - consider optimization`);
|
||||
}
|
||||
|
||||
if (m.query.buildTime > 100) {
|
||||
issues.push(`Slow query build (${m.query.buildTime}ms) - entity lexicon may be too large`);
|
||||
}
|
||||
|
||||
if (m.evidence.l1CosineTime > 1000) {
|
||||
issues.push(`Slow L1 cosine scoring (${m.evidence.l1CosineTime}ms) - too many chunks pulled`);
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────
|
||||
// Diffusion 问题
|
||||
// ─────────────────────────────────────────────────────────────────
|
||||
|
||||
if (m.diffusion.graphEdges === 0 && m.diffusion.seedCount > 0) {
|
||||
issues.push('No diffusion graph edges - atoms may lack edges fields');
|
||||
}
|
||||
|
||||
if (m.diffusion.pprActivated > 0 && m.diffusion.cosineGatePassed === 0) {
|
||||
issues.push('All PPR-activated nodes failed cosine gate - graph structure diverged from query semantics');
|
||||
}
|
||||
|
||||
m.quality.diffusionEffectiveRate = m.diffusion.pprActivated > 0
|
||||
? Math.round((m.diffusion.finalCount / m.diffusion.pprActivated) * 100)
|
||||
: 0;
|
||||
|
||||
if (m.diffusion.cosineGateNoVector > 5) {
|
||||
issues.push(`${m.diffusion.cosineGateNoVector} PPR nodes missing vectors - L0 vectorization may be incomplete`);
|
||||
}
|
||||
|
||||
if (m.diffusion.time > 50) {
|
||||
issues.push(`Slow diffusion (${m.diffusion.time}ms) - graph may be too dense`);
|
||||
}
|
||||
|
||||
if (m.diffusion.pprActivated > 0 && (m.diffusion.postGatePassRate < 20 || m.diffusion.postGatePassRate > 60)) {
|
||||
issues.push(`Diffusion post-gate pass rate out of target (${m.diffusion.postGatePassRate}%)`);
|
||||
}
|
||||
|
||||
return issues;
|
||||
}
|
||||
387
modules/story-summary/vector/retrieval/query-builder.js
Normal file
387
modules/story-summary/vector/retrieval/query-builder.js
Normal file
@@ -0,0 +1,387 @@
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
// query-builder.js - 确定性查询构建器（无 LLM）
|
||||
//
|
||||
// 职责：
|
||||
// 1. 从最近 3 条消息构建 QueryBundle（加权向量段）
|
||||
// 2. 用第一轮召回结果产出 hints 段用于 R2 增强
|
||||
//
|
||||
// 加权向量设计：
|
||||
// - 每条消息独立 embed，得到独立向量
|
||||
// - 按位置分配基础权重（焦点 > 近上下文 > 远上下文）
|
||||
// - 短消息通过 lengthFactor 自动降权（下限 35%）
|
||||
// - recall.js 负责 embed + 归一化 + 加权平均
|
||||
//
|
||||
// 焦点确定：
|
||||
// - pendingUserMessage 存在 → 它是焦点
|
||||
// - 否则 → lastMessages 最后一条是焦点
|
||||
//
|
||||
// 不负责：向量化、检索、rerank
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
|
||||
import { getContext } from '../../../../../../../extensions.js';
|
||||
import { buildEntityLexicon, buildDisplayNameMap, extractEntitiesFromText, buildCharacterPools } from './entity-lexicon.js';
|
||||
import { getSummaryStore } from '../../data/store.js';
|
||||
import { filterText } from '../utils/text-filter.js';
|
||||
import { tokenizeForIndex as tokenizerTokenizeForIndex } from '../utils/tokenizer.js';
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
// 权重常量
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
|
||||
// R1 基础权重：[...context(oldest→newest), focus]
|
||||
// 焦点消息占 55%，最近上下文 30%，更早上下文 15%
|
||||
export const FOCUS_BASE_WEIGHT = 0.55;
|
||||
export const CONTEXT_BASE_WEIGHTS = [0.15, 0.30];
|
||||
|
||||
// R2 基础权重：焦点让权给 hints
|
||||
export const FOCUS_BASE_WEIGHT_R2 = 0.45;
|
||||
export const CONTEXT_BASE_WEIGHTS_R2 = [0.10, 0.20];
|
||||
export const HINTS_BASE_WEIGHT = 0.25;
|
||||
|
||||
// 长度惩罚：< 50 字线性衰减，下限 35%
|
||||
export const LENGTH_FULL_THRESHOLD = 50;
|
||||
export const LENGTH_MIN_FACTOR = 0.35;
|
||||
// 归一化后的焦点最小占比（由 recall.js 在归一化后硬保底）
|
||||
// 语义：即使焦点文本很短，也不能被稀释到过低权重
|
||||
export const FOCUS_MIN_NORMALIZED_WEIGHT = 0.35;
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
// 其他常量
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
|
||||
const MEMORY_HINT_ATOMS_MAX = 5;
|
||||
const MEMORY_HINT_EVENTS_MAX = 3;
|
||||
const LEXICAL_TERMS_MAX = 10;
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
// 工具函数
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* 清洗消息文本（与 chunk-builder / recall 保持一致）
|
||||
* @param {string} text
|
||||
* @returns {string}
|
||||
*/
|
||||
function cleanMessageText(text) {
|
||||
return filterText(text)
|
||||
.replace(/\[tts:[^\]]*\]/gi, '')
|
||||
.replace(/<state>[\s\S]*?<\/state>/gi, '')
|
||||
.trim();
|
||||
}
|
||||
|
||||
/**
|
||||
* 清理事件摘要（移除楼层标记）
|
||||
* @param {string} summary
|
||||
* @returns {string}
|
||||
*/
|
||||
function cleanSummary(summary) {
|
||||
return String(summary || '')
|
||||
.replace(/\s*\(#\d+(?:-\d+)?\)\s*$/, '')
|
||||
.trim();
|
||||
}
|
||||
|
||||
/**
|
||||
* 计算长度因子
|
||||
*
|
||||
* charCount >= 50 → 1.0
|
||||
* charCount = 0 → 0.35
|
||||
* 中间线性插值
|
||||
*
|
||||
* @param {number} charCount - 清洗后内容字符数（不含 speaker 前缀）
|
||||
* @returns {number} 0.35 ~ 1.0
|
||||
*/
|
||||
export function computeLengthFactor(charCount) {
|
||||
if (charCount >= LENGTH_FULL_THRESHOLD) return 1.0;
|
||||
if (charCount <= 0) return LENGTH_MIN_FACTOR;
|
||||
return LENGTH_MIN_FACTOR + (1.0 - LENGTH_MIN_FACTOR) * (charCount / LENGTH_FULL_THRESHOLD);
|
||||
}
|
||||
|
||||
/**
|
||||
* 从文本中提取高频实词（用于词法检索）
|
||||
*
|
||||
* @param {string} text - 清洗后的文本
|
||||
* @param {number} maxTerms - 最大词数
|
||||
* @returns {string[]}
|
||||
*/
|
||||
function extractKeyTerms(text, maxTerms = LEXICAL_TERMS_MAX) {
|
||||
if (!text) return [];
|
||||
|
||||
const tokens = tokenizerTokenizeForIndex(text);
|
||||
const freq = new Map();
|
||||
for (const token of tokens) {
|
||||
const key = String(token || '').toLowerCase();
|
||||
if (!key) continue;
|
||||
freq.set(key, (freq.get(key) || 0) + 1);
|
||||
}
|
||||
|
||||
return Array.from(freq.entries())
|
||||
.sort((a, b) => b[1] - a[1])
|
||||
.slice(0, maxTerms)
|
||||
.map(([term]) => term);
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
// 类型定义
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* @typedef {object} QuerySegment
|
||||
* @property {string} text - 待 embed 的文本（含 speaker 前缀，纯自然语言）
|
||||
* @property {number} baseWeight - R1 基础权重
|
||||
* @property {number} charCount - 内容字符数（不含 speaker 前缀，用于 lengthFactor）
|
||||
*/
|
||||
|
||||
/**
|
||||
* @typedef {object} QueryBundle
|
||||
* @property {QuerySegment[]} querySegments - R1 向量段（上下文 oldest→newest，焦点在末尾）
|
||||
* @property {QuerySegment|null} hintsSegment - R2 hints 段（refinement 后填充）
|
||||
* @property {string} rerankQuery - rerank 用的纯自然语言查询（焦点在前）
|
||||
* @property {string[]} lexicalTerms - MiniSearch 查询词
|
||||
* @property {string[]} focusTerms - 焦点词（原 focusEntities）
|
||||
* @property {string[]} focusCharacters - 焦点人物（focusTerms ∩ trustedCharacters）
|
||||
* @property {string[]} focusEntities - Deprecated alias of focusTerms
|
||||
* @property {Set<string>} allEntities - Full entity lexicon (includes non-character entities)
|
||||
* @property {Set<string>} allCharacters - Union of trusted and candidate character pools
|
||||
* @property {Set<string>} trustedCharacters - Clean character pool (main/arcs/name2/L2 participants)
|
||||
* @property {Set<string>} candidateCharacters - Extended character pool from L0 edges.s/t after cleanup
|
||||
* @property {Set<string>} _lexicon - 实体词典（内部使用）
|
||||
* @property {Map<string, string>} _displayMap - 标准化→原词形映射（内部使用）
|
||||
*/
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
// 内部：消息条目构建
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* @typedef {object} MessageEntry
|
||||
* @property {string} text - speaker：内容（完整文本）
|
||||
* @property {number} charCount - 内容字符数（不含 speaker 前缀）
|
||||
*/
|
||||
|
||||
/**
|
||||
* 清洗消息并构建条目
|
||||
* @param {object} message - chat 消息对象
|
||||
* @param {object} context - { name1, name2 }
|
||||
* @returns {MessageEntry|null}
|
||||
*/
|
||||
function buildMessageEntry(message, context) {
|
||||
if (!message?.mes) return null;
|
||||
|
||||
const speaker = message.is_user
|
||||
? (context.name1 || '用户')
|
||||
: (message.name || context.name2 || '角色');
|
||||
|
||||
const clean = cleanMessageText(message.mes);
|
||||
if (!clean) return null;
|
||||
|
||||
return {
|
||||
text: `${speaker}：${clean}`,
|
||||
charCount: clean.length,
|
||||
};
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
// 阶段 1：构建 QueryBundle
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* 构建初始查询包
|
||||
*
|
||||
* 消息布局（K=3 时）：
|
||||
* msg[0] = USER(#N-2) 上下文 baseWeight = 0.15
|
||||
* msg[1] = AI(#N-1) 上下文 baseWeight = 0.30
|
||||
* msg[2] = USER(#N) 焦点 baseWeight = 0.55
|
||||
*
|
||||
* 焦点确定：
|
||||
* pendingUserMessage 存在 → 焦点，所有 lastMessages 为上下文
|
||||
* pendingUserMessage 不存在 → lastMessages[-1] 为焦点，其余为上下文
|
||||
*
|
||||
* @param {object[]} lastMessages - 最近 K 条消息（由 recall.js 传入）
|
||||
* @param {string|null} pendingUserMessage - 用户刚输入但未进 chat 的消息
|
||||
* @param {object|null} store
|
||||
* @param {object|null} context - { name1, name2 }
|
||||
* @returns {QueryBundle}
|
||||
*/
|
||||
export function buildQueryBundle(lastMessages, pendingUserMessage, store = null, context = null) {
|
||||
if (!store) store = getSummaryStore();
|
||||
if (!context) {
|
||||
const ctx = getContext();
|
||||
context = { name1: ctx.name1, name2: ctx.name2 };
|
||||
}
|
||||
|
||||
// 1. 实体/人物词典
|
||||
const lexicon = buildEntityLexicon(store, context);
|
||||
const displayMap = buildDisplayNameMap(store, context);
|
||||
const { trustedCharacters, candidateCharacters, allCharacters } = buildCharacterPools(store, context);
|
||||
|
||||
// 2. 分离焦点与上下文
|
||||
const contextEntries = [];
|
||||
let focusEntry = null;
|
||||
const allCleanTexts = [];
|
||||
|
||||
if (pendingUserMessage) {
|
||||
// pending 是焦点，所有 lastMessages 是上下文
|
||||
const pendingClean = cleanMessageText(pendingUserMessage);
|
||||
if (pendingClean) {
|
||||
const speaker = context.name1 || '用户';
|
||||
focusEntry = {
|
||||
text: `${speaker}：${pendingClean}`,
|
||||
charCount: pendingClean.length,
|
||||
};
|
||||
allCleanTexts.push(pendingClean);
|
||||
}
|
||||
|
||||
for (const m of (lastMessages || [])) {
|
||||
const entry = buildMessageEntry(m, context);
|
||||
if (entry) {
|
||||
contextEntries.push(entry);
|
||||
allCleanTexts.push(cleanMessageText(m.mes));
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// 无 pending → lastMessages[-1] 是焦点
|
||||
const msgs = lastMessages || [];
|
||||
|
||||
if (msgs.length > 0) {
|
||||
const lastMsg = msgs[msgs.length - 1];
|
||||
const entry = buildMessageEntry(lastMsg, context);
|
||||
if (entry) {
|
||||
focusEntry = entry;
|
||||
allCleanTexts.push(cleanMessageText(lastMsg.mes));
|
||||
}
|
||||
}
|
||||
|
||||
for (let i = 0; i < msgs.length - 1; i++) {
|
||||
const entry = buildMessageEntry(msgs[i], context);
|
||||
if (entry) {
|
||||
contextEntries.push(entry);
|
||||
allCleanTexts.push(cleanMessageText(msgs[i].mes));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 3. 提取焦点词与焦点人物
|
||||
const combinedText = allCleanTexts.join(' ');
|
||||
const focusTerms = extractEntitiesFromText(combinedText, lexicon, displayMap);
|
||||
const focusCharacters = focusTerms.filter(term => trustedCharacters.has(term.toLowerCase()));
|
||||
|
||||
// 4. 构建 querySegments
|
||||
// 上下文在前（oldest → newest），焦点在末尾
|
||||
// 上下文权重从 CONTEXT_BASE_WEIGHTS 尾部对齐分配
|
||||
const querySegments = [];
|
||||
|
||||
for (let i = 0; i < contextEntries.length; i++) {
|
||||
const weightIdx = Math.max(0, CONTEXT_BASE_WEIGHTS.length - contextEntries.length + i);
|
||||
querySegments.push({
|
||||
text: contextEntries[i].text,
|
||||
baseWeight: CONTEXT_BASE_WEIGHTS[weightIdx] || CONTEXT_BASE_WEIGHTS[0],
|
||||
charCount: contextEntries[i].charCount,
|
||||
});
|
||||
}
|
||||
|
||||
if (focusEntry) {
|
||||
querySegments.push({
|
||||
text: focusEntry.text,
|
||||
baseWeight: FOCUS_BASE_WEIGHT,
|
||||
charCount: focusEntry.charCount,
|
||||
});
|
||||
}
|
||||
|
||||
// 5. rerankQuery（焦点在前，纯自然语言，无前缀）
|
||||
const contextLines = contextEntries.map(e => e.text);
|
||||
const rerankQuery = focusEntry
|
||||
? [focusEntry.text, ...contextLines].join('\n')
|
||||
: contextLines.join('\n');
|
||||
|
||||
// 6. lexicalTerms（实体优先 + 高频实词补充）
|
||||
const entityTerms = focusTerms.map(e => e.toLowerCase());
|
||||
const textTerms = extractKeyTerms(combinedText);
|
||||
const termSet = new Set(entityTerms);
|
||||
for (const t of textTerms) {
|
||||
if (termSet.size >= LEXICAL_TERMS_MAX) break;
|
||||
termSet.add(t);
|
||||
}
|
||||
|
||||
return {
|
||||
querySegments,
|
||||
hintsSegment: null,
|
||||
rerankQuery,
|
||||
lexicalTerms: Array.from(termSet),
|
||||
focusTerms,
|
||||
focusCharacters,
|
||||
focusEntities: focusTerms, // deprecated alias (compat)
|
||||
allEntities: lexicon,
|
||||
allCharacters,
|
||||
trustedCharacters,
|
||||
candidateCharacters,
|
||||
_lexicon: lexicon,
|
||||
_displayMap: displayMap,
|
||||
};
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
// 阶段 3：Query Refinement（用第一轮召回结果产出 hints 段）
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* 用第一轮召回结果增强 QueryBundle
|
||||
*
|
||||
* 原地修改 bundle（仅 query/rerank 辅助项）：
|
||||
* - hintsSegment：填充 hints 段（供 R2 加权使用）
|
||||
* - lexicalTerms：可能追加 hints 中的关键词
|
||||
* - rerankQuery：不变（保持焦点优先的纯自然语言）
|
||||
*
|
||||
* @param {QueryBundle} bundle - 原始查询包
|
||||
* @param {object[]} anchorHits - 第一轮 L0 命中（按相似度降序）
|
||||
* @param {object[]} eventHits - 第一轮 L2 命中（按相似度降序）
|
||||
*/
|
||||
export function refineQueryBundle(bundle, anchorHits, eventHits) {
|
||||
const hints = [];
|
||||
|
||||
// 1. 从 top anchorHits 提取 memory hints
|
||||
const topAnchors = (anchorHits || []).slice(0, MEMORY_HINT_ATOMS_MAX);
|
||||
for (const hit of topAnchors) {
|
||||
const semantic = hit.atom?.semantic || '';
|
||||
if (semantic) hints.push(semantic);
|
||||
}
|
||||
|
||||
// 2. 从 top eventHits 提取 memory hints
|
||||
const topEvents = (eventHits || []).slice(0, MEMORY_HINT_EVENTS_MAX);
|
||||
for (const hit of topEvents) {
|
||||
const ev = hit.event || {};
|
||||
const title = String(ev.title || '').trim();
|
||||
const summary = cleanSummary(ev.summary);
|
||||
const line = title && summary
|
||||
? `${title}: ${summary}`
|
||||
: title || summary;
|
||||
if (line) hints.push(line);
|
||||
}
|
||||
|
||||
// 3. 构建 hintsSegment
|
||||
if (hints.length > 0) {
|
||||
const hintsText = hints.join('\n');
|
||||
bundle.hintsSegment = {
|
||||
text: hintsText,
|
||||
baseWeight: HINTS_BASE_WEIGHT,
|
||||
charCount: hintsText.length,
|
||||
};
|
||||
} else {
|
||||
bundle.hintsSegment = null;
|
||||
}
|
||||
|
||||
// 4. rerankQuery 不变
|
||||
// cross-encoder 接收纯自然语言 query，不受 hints 干扰
|
||||
|
||||
// 5. 增强 lexicalTerms
|
||||
if (hints.length > 0) {
|
||||
const hintTerms = extractKeyTerms(hints.join(' '), 5);
|
||||
const termSet = new Set(bundle.lexicalTerms);
|
||||
for (const t of hintTerms) {
|
||||
if (termSet.size >= LEXICAL_TERMS_MAX) break;
|
||||
if (!termSet.has(t)) {
|
||||
termSet.add(t);
|
||||
bundle.lexicalTerms.push(t);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
1399
modules/story-summary/vector/retrieval/recall.js
Normal file
1399
modules/story-summary/vector/retrieval/recall.js
Normal file
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user