Update retrieval, rerank, and indexing changes

modules/story-summary/vector/llm/reranker.js

@@ -1,4 +1,4 @@
-// ═══════════════════════════════════════════════════════════════════════════
+// ═══════════════════════════════════════════════════════════════════════════
 // Reranker - 硅基 bge-reranker-v2-m3
 // 对候选文档进行精排，过滤与 query 不相关的内容
 // ═══════════════════════════════════════════════════════════════════════════
@@ -11,6 +11,8 @@ const RERANK_URL = 'https://api.siliconflow.cn/v1/rerank';
 const RERANK_MODEL = 'BAAI/bge-reranker-v2-m3';
 const DEFAULT_TIMEOUT = 15000;
 const MAX_DOCUMENTS = 100;  // API 限制
+const RERANK_BATCH_SIZE = 20;
+const RERANK_MAX_CONCURRENCY = 5;
 
 /**
  * 对文档列表进行 Rerank 精排
@@ -140,25 +142,83 @@ export async function rerankChunks(query, chunks, options = {}) {
     const { topN = 40, minScore = 0.1 } = options;
 
     if (!chunks?.length) return [];
-    if (chunks.length <= topN) {
-        const texts = chunks.map(c => c.text || c.semantic || '');
-        const { results, failed } = await rerank(query, texts, { topN: chunks.length, ...options });
+
+    const texts = chunks.map(c => c.text || c.semantic || '');
+
+    // ─── 单批：直接调用 ───
+    if (texts.length <= RERANK_BATCH_SIZE) {
+        const { results, failed } = await rerank(query, texts, {
+            topN: Math.min(topN, texts.length),
+            timeout: options.timeout,
+            signal: options.signal,
+        });
 
         if (failed) {
             return chunks.map(c => ({ ...c, _rerankScore: 0, _rerankFailed: true }));
         }
 
-        const scoreMap = new Map(results.map(r => [r.index, r.relevance_score]));
-        return chunks.map((c, i) => ({
-            ...c,
-            _rerankScore: scoreMap.get(i) ?? 0,
-        })).sort((a, b) => b._rerankScore - a._rerankScore);
+        return results
+            .filter(r => r.relevance_score >= minScore)
+            .sort((a, b) => b.relevance_score - a.relevance_score)
+            .slice(0, topN)
+            .map(r => ({
+                ...chunks[r.index],
+                _rerankScore: r.relevance_score,
+            }));
     }
 
-    const texts = chunks.map(c => c.text || c.semantic || '');
-    const { results, failed } = await rerank(query, texts, { topN, ...options });
+    // ─── 多批：拆分 → 并发 → 合并 ───
+    const batches = [];
+    for (let i = 0; i < texts.length; i += RERANK_BATCH_SIZE) {
+        batches.push({
+            texts: texts.slice(i, i + RERANK_BATCH_SIZE),
+            offset: i,
+        });
+    }
 
-    if (failed) {
+    const concurrency = Math.min(batches.length, RERANK_MAX_CONCURRENCY);
+    xbLog.info(MODULE_ID, `并发 Rerank: ${batches.length} 批 × ≤${RERANK_BATCH_SIZE} docs, concurrency=${concurrency}`);
+
+    const batchResults = new Array(batches.length);
+    let failedBatches = 0;
+
+    const runBatch = async (batchIdx) => {
+        const batch = batches[batchIdx];
+        const { results, failed } = await rerank(query, batch.texts, {
+            topN: batch.texts.length,
+            timeout: options.timeout,
+            signal: options.signal,
+        });
+
+        if (failed) {
+            failedBatches++;
+            // 单批降级：保留原始顺序，score=0
+            batchResults[batchIdx] = batch.texts.map((_, i) => ({
+                globalIndex: batch.offset + i,
+                relevance_score: 0,
+                _batchFailed: true,
+            }));
+        } else {
+            batchResults[batchIdx] = results.map(r => ({
+                globalIndex: batch.offset + r.index,
+                relevance_score: r.relevance_score,
+            }));
+        }
+    };
+
+    // 并发池
+    let nextIdx = 0;
+    const worker = async () => {
+        while (nextIdx < batches.length) {
+            const idx = nextIdx++;
+            await runBatch(idx);
+        }
+    };
+    await Promise.all(Array.from({ length: concurrency }, () => worker()));
+
+    // 全部失败 → 整体降级
+    if (failedBatches === batches.length) {
+        xbLog.warn(MODULE_ID, `全部 ${batches.length} 批 rerank 失败，整体降级`);
         return chunks.slice(0, topN).map(c => ({
             ...c,
             _rerankScore: 0,
@@ -166,15 +226,25 @@ export async function rerankChunks(query, chunks, options = {}) {
         }));
     }
 
-    return results
-        .filter(r => r.relevance_score >= minScore)
-        .sort((a, b) => b.relevance_score - a.relevance_score)
-        .map(r => ({
-            ...chunks[r.index],
-            _rerankScore: r.relevance_score,
-        }));
-}
+    // 合并所有批次结果
+    const merged = batchResults.flat();
 
+    const selected = merged
+        .filter(r => r._batchFailed || r.relevance_score >= minScore)
+        .sort((a, b) => b.relevance_score - a.relevance_score)
+        .slice(0, topN)
+        .map(r => ({
+            ...chunks[r.globalIndex],
+            _rerankScore: r.relevance_score,
+            ...(r._batchFailed ? { _rerankFailed: true } : {}),
+        }));
+
+    xbLog.info(MODULE_ID,
+        `Rerank 合并: ${merged.length} candidates, ${failedBatches}/${batches.length} 批失败, 选中 ${selected.length}`
+    );
+
+    return selected;
+}
 /**
  * 测试 Rerank 服务连接
  */