|
|
|
@ -1,38 +1,57 @@ |
|
|
|
package com.wok.supportbot.service; |
|
|
|
|
|
|
|
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper; |
|
|
|
import com.wok.supportbot.dao.KnowledgeDocumentMapper; |
|
|
|
import com.wok.supportbot.document.transform.MyKeywordEnricher; |
|
|
|
import com.wok.supportbot.document.transform.MyTokenTextSplitter; |
|
|
|
import com.wok.supportbot.entity.KnowledgeDocument; |
|
|
|
import lombok.extern.slf4j.Slf4j; |
|
|
|
import org.springframework.ai.document.Document; |
|
|
|
import org.springframework.ai.vectorstore.VectorStore; |
|
|
|
import org.springframework.beans.factory.annotation.Autowired; |
|
|
|
import org.springframework.beans.factory.annotation.Value; |
|
|
|
import org.springframework.jdbc.core.JdbcTemplate; |
|
|
|
import org.springframework.scheduling.annotation.Async; |
|
|
|
import org.springframework.stereotype.Service; |
|
|
|
|
|
|
|
import java.util.ArrayList; |
|
|
|
import java.util.HashMap; |
|
|
|
import java.util.List; |
|
|
|
import java.util.Map; |
|
|
|
import java.util.function.Supplier; |
|
|
|
import java.util.stream.Collectors; |
|
|
|
|
|
|
|
/** |
|
|
|
* 文档异步处理服务 |
|
|
|
* 负责文档的分块、关键词提取、向量化等耗时操作，在后台线程中执行 |
|
|
|
* 负责文档的分块与向量化（按批隔离、失败可感知、有限自动重试），在后台线程中执行 |
|
|
|
*/ |
|
|
|
@Service |
|
|
|
@Slf4j |
|
|
|
public class DocumentProcessingService { |
|
|
|
|
|
|
|
/** 一批向量化的分块数：控制单批请求体积与"炸点"范围（含 volcengine-vision 逐条串行路径） */ |
|
|
|
@Value("${knowledge.vector.batch-size:50}") |
|
|
|
private int embedBatchSize = 50; |
|
|
|
|
|
|
|
/** |
|
|
|
* 文档级自动重试次数（不含首次尝试）。 |
|
|
|
* 注意：批内 EmbeddingModel 已自带 3 次指数退避重试（EmbeddingModelFactory.createRetryTemplate）， |
|
|
|
* 文档级重试叠加在最外层；文档级只对"整篇仍有失败批"触发，多数瞬时错误已在内层耗掉，外层命中率低。 |
|
|
|
* 若厂商限流严重，可调小本值或调大 RETRY_BASE_DELAY_MS 冷却时间 |
|
|
|
*/ |
|
|
|
private static final int MAX_DOC_LEVEL_RETRIES = 2; |
|
|
|
|
|
|
|
/** 重试等待基数（ms），指数放大（2s → 4s） */ |
|
|
|
private static final long RETRY_BASE_DELAY_MS = 2_000L; |
|
|
|
|
|
|
|
/** 连续 N 批失败即中止本轮，交给文档级重试，避免打爆全部批次 */ |
|
|
|
private static final int MAX_CONSECUTIVE_BATCH_FAILURES = 3; |
|
|
|
|
|
|
|
@Autowired |
|
|
|
private KnowledgeDocumentMapper documentMapper; |
|
|
|
|
|
|
|
@Autowired |
|
|
|
private MyTokenTextSplitter myTokenTextSplitter; |
|
|
|
|
|
|
|
@Autowired |
|
|
|
private MyKeywordEnricher myKeywordEnricher; |
|
|
|
|
|
|
|
@Autowired |
|
|
|
private VectorStore pgVectorVectorStore; |
|
|
|
|
|
|
|
@ -40,8 +59,8 @@ public class DocumentProcessingService { |
|
|
|
private JdbcTemplate jdbcTemplate; |
|
|
|
|
|
|
|
/** |
|
|
|
* 异步处理文档：分块 → 关键词提取 → 向量化 → 更新状态 |
|
|
|
* 不加跨方法事务：AI 关键词提取与向量化属于慢速网络调用，期间不占用数据库连接， |
|
|
|
* 异步处理文档：分块 → 向量化（分批入库）→ 更新状态 |
|
|
|
* 不加跨方法事务：向量化属于慢速网络调用，期间不占用数据库连接， |
|
|
|
* 避免文件夹批量上传时多个异步任务把连接池占满导致连接超时。 |
|
|
|
* |
|
|
|
* @param docId 文档ID |
|
|
|
@ -64,52 +83,16 @@ public class DocumentProcessingService { |
|
|
|
return; |
|
|
|
} |
|
|
|
|
|
|
|
try { |
|
|
|
// 1. 分块处理（使用 per-doc 参数或全局配置） |
|
|
|
List<Document> splitDocuments = myTokenTextSplitter.splitDocuments(documents, chunkSize, overlap); |
|
|
|
DocMeta meta = DocMeta.of(docId, sourceName, title, categoryId, tags, |
|
|
|
Boolean.TRUE.equals(doc.getEnabled())); |
|
|
|
|
|
|
|
// 2. 为每个分块设置 metadata |
|
|
|
for (int i = 0; i < splitDocuments.size(); i++) { |
|
|
|
Document d = splitDocuments.get(i); |
|
|
|
Map<String, Object> meta = new HashMap<>(d.getMetadata()); |
|
|
|
meta.put("documentId", String.valueOf(docId)); |
|
|
|
meta.put("chunkIndex", i); |
|
|
|
meta.put("sourceName", sourceName); |
|
|
|
meta.put("title", title); |
|
|
|
if (categoryId != null && categoryId > 0) { |
|
|
|
meta.put("categoryId", String.valueOf(categoryId)); |
|
|
|
} |
|
|
|
if (tags != null && !tags.isEmpty()) { |
|
|
|
meta.put("tags", tags); |
|
|
|
} |
|
|
|
// P1-2.1: 标记启用状态，用于 RAG 检索过滤 |
|
|
|
meta.put("enabled", String.valueOf(Boolean.TRUE.equals(doc.getEnabled()))); |
|
|
|
splitDocuments.set(i, new Document(d.getId(), d.getText(), meta)); |
|
|
|
} |
|
|
|
|
|
|
|
// 3. 关键词提取 |
|
|
|
List<Document> enrichedDocuments = myKeywordEnricher.enrichDocuments(splitDocuments); |
|
|
|
|
|
|
|
// 4. 向量化存储 |
|
|
|
pgVectorVectorStore.add(enrichedDocuments); |
|
|
|
|
|
|
|
// 5. 更新状态为 READY |
|
|
|
doc.setStatus("READY"); |
|
|
|
doc.setChunkCount(enrichedDocuments.size()); |
|
|
|
documentMapper.updateById(doc); |
|
|
|
|
|
|
|
log.info("异步处理文档完成: id={}, title={}, chunks={}", docId, title, enrichedDocuments.size()); |
|
|
|
|
|
|
|
} catch (Exception e) { |
|
|
|
doc.setStatus("FAILED"); |
|
|
|
doc.setErrorMessage(e.getMessage()); |
|
|
|
documentMapper.updateById(doc); |
|
|
|
log.error("异步处理文档失败: id={}, title={}", docId, title, e); |
|
|
|
} |
|
|
|
// 新文档首次处理无需清理（尚无向量）；若中途失败触发整体重试，框架内会先清残留向量再重建 |
|
|
|
runPipelineWithRetry(doc, meta, false, |
|
|
|
() -> myTokenTextSplitter.splitDocuments(documents, chunkSize, overlap)); |
|
|
|
} |
|
|
|
|
|
|
|
/** |
|
|
|
* 异步重新处理文档（重新分块 + 向量化） |
|
|
|
* 异步重新处理文档（清理旧向量后按批重建向量化） |
|
|
|
* 使用文档存储的 per-doc 分块参数（extraConfig），如无则使用全局配置 |
|
|
|
* |
|
|
|
* @param docId 文档ID |
|
|
|
@ -124,61 +107,265 @@ public class DocumentProcessingService { |
|
|
|
return; |
|
|
|
} |
|
|
|
|
|
|
|
try { |
|
|
|
// 删除旧向量 |
|
|
|
String sql = "SELECT id::text FROM vector_store WHERE metadata->>'documentId' = ?"; |
|
|
|
List<String> oldIds = jdbcTemplate.queryForList(sql, String.class, String.valueOf(docId)); |
|
|
|
if (!oldIds.isEmpty()) { |
|
|
|
pgVectorVectorStore.delete(oldIds); |
|
|
|
} |
|
|
|
DocMeta meta = DocMeta.fromDocument(doc); |
|
|
|
|
|
|
|
// 从 extraConfig 读取 per-doc 分块参数 |
|
|
|
// reprocess 兼容：首轮就清理旧向量后按批重建；分块参数从 extraConfig 读取 |
|
|
|
runPipelineWithRetry(doc, meta, true, () -> { |
|
|
|
Integer chunkSize = null; |
|
|
|
Integer overlap = null; |
|
|
|
if (doc.getExtraConfig() != null) { |
|
|
|
Object cs = doc.getExtraConfig().get("chunkSize"); |
|
|
|
Object ol = doc.getExtraConfig().get("overlap"); |
|
|
|
if (cs instanceof Number) chunkSize = ((Number) cs).intValue(); |
|
|
|
if (ol instanceof Number) overlap = ((Number) ol).intValue(); |
|
|
|
if (doc.getExtraConfig().get("chunkSize") instanceof Number cs) chunkSize = cs.intValue(); |
|
|
|
if (doc.getExtraConfig().get("overlap") instanceof Number ol) overlap = ol.intValue(); |
|
|
|
} |
|
|
|
return myTokenTextSplitter.splitDocuments(documents, chunkSize, overlap); |
|
|
|
}); |
|
|
|
} |
|
|
|
|
|
|
|
/** |
|
|
|
* 文档级处理框架（外层兜底）：切分 → 分批向量化 → 失败自动整体重试有限次数。 |
|
|
|
* 外层 try-catch 保证任何未预期异常下文档状态都从 PROCESSING 收敛到 FAILED， |
|
|
|
* 避免 @Async 异常被 Spring 静默吞掉导致文档永久悬挂（前端轮询永不结束） |
|
|
|
*/ |
|
|
|
private void runPipelineWithRetry(KnowledgeDocument doc, DocMeta meta, |
|
|
|
boolean cleanBeforeFirstAttempt, |
|
|
|
Supplier<List<Document>> splitter) { |
|
|
|
try { |
|
|
|
doPipelineWithRetry(doc, meta, cleanBeforeFirstAttempt, splitter); |
|
|
|
} catch (Exception e) { |
|
|
|
// 兜底落 FAILED；若状态写入也失败（如 DB 抖动），仅记日志，避免再次上抛造成悬挂 |
|
|
|
log.error("文档向量化处理出现未捕获异常: id={}", doc.getId(), e); |
|
|
|
try { |
|
|
|
String reason = e.getMessage() == null |
|
|
|
? e.getClass().getSimpleName() |
|
|
|
: truncate(e.getMessage()); |
|
|
|
patchStatus(doc.getId(), "FAILED", safeCountVectors(String.valueOf(doc.getId())), |
|
|
|
"内部错误: " + reason); |
|
|
|
} catch (Exception ex) { |
|
|
|
log.error("兜底标记文档处理失败状态也失败: id={}, error={}", doc.getId(), ex.getMessage(), ex); |
|
|
|
} |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
private void doPipelineWithRetry(KnowledgeDocument doc, DocMeta meta, |
|
|
|
boolean cleanBeforeFirstAttempt, |
|
|
|
Supplier<List<Document>> splitter) { |
|
|
|
String docIdStr = String.valueOf(doc.getId()); |
|
|
|
|
|
|
|
for (int attempt = 0; attempt <= MAX_DOC_LEVEL_RETRIES; attempt++) { |
|
|
|
// 每次尝试前复查文档仍存在（用户可能在重试等待/sleep 期间删除文档），避免向已删文档写孤儿向量 |
|
|
|
if (documentMapper.selectById(doc.getId()) == null) { |
|
|
|
log.info("文档在处理期间已被删除，放弃处理: id={}", doc.getId()); |
|
|
|
return; |
|
|
|
} |
|
|
|
|
|
|
|
// 重试前：指数等待 + 清理上一轮残留向量，避免 split 重建产生新 Document.id 造成重复向量 |
|
|
|
if (attempt > 0) { |
|
|
|
if (!sleepBackoff(attempt)) { |
|
|
|
patchStatus(doc.getId(), "FAILED", safeCountVectors(docIdStr), "处理被中断"); |
|
|
|
return; |
|
|
|
} |
|
|
|
deleteVectorsByDocumentId(docIdStr); |
|
|
|
log.warn("文档向量化整体重试第 {}/{} 次: id={}", attempt, MAX_DOC_LEVEL_RETRIES, doc.getId()); |
|
|
|
} else if (cleanBeforeFirstAttempt) { |
|
|
|
deleteVectorsByDocumentId(docIdStr); |
|
|
|
} |
|
|
|
|
|
|
|
// 重新分块（使用 per-doc 参数或全局配置） |
|
|
|
List<Document> splitDocuments = myTokenTextSplitter.splitDocuments(documents, chunkSize, overlap); |
|
|
|
List<Document> chunks; |
|
|
|
try { |
|
|
|
chunks = splitter.get(); |
|
|
|
} catch (Exception e) { |
|
|
|
if (attempt < MAX_DOC_LEVEL_RETRIES && isRetryable(e.getMessage())) { |
|
|
|
continue; |
|
|
|
} |
|
|
|
markFailed(doc, "文档分块失败: " + truncate(e.getMessage())); |
|
|
|
return; |
|
|
|
} |
|
|
|
|
|
|
|
if (chunks == null || chunks.isEmpty()) { |
|
|
|
// 分块为空属于内容/参数问题，重试不会自愈 |
|
|
|
markFailed(doc, "文档分块结果为空（无可向量化文本或小于最小分块），请检查内容与分块参数"); |
|
|
|
return; |
|
|
|
} |
|
|
|
|
|
|
|
List<Document> annotated = annotateChunks(chunks, meta); |
|
|
|
BatchResult result = vectorizeInBatches(docIdStr, annotated, doc.getId()); |
|
|
|
|
|
|
|
// 处理期间文档被删除：终止且不再写状态（删除是用户明确意图，不覆盖为 FAILED） |
|
|
|
if (result.cancelled) { |
|
|
|
return; |
|
|
|
} |
|
|
|
|
|
|
|
int stored = safeCountVectors(docIdStr); |
|
|
|
if (!result.hasFailure()) { |
|
|
|
patchStatus(doc.getId(), "READY", stored, null); |
|
|
|
log.info("文档向量化成功: id={}, title={}, chunks={}", doc.getId(), doc.getTitle(), stored); |
|
|
|
return; |
|
|
|
} |
|
|
|
|
|
|
|
// 有失败批：若仍有余量且错误属瞬时/基础设施类，则整篇重试 |
|
|
|
if (attempt < MAX_DOC_LEVEL_RETRIES && isRetryable(result.firstError)) { |
|
|
|
log.warn("存在失败批，准备整体重试: id={}, total={}, stored={}, firstError={}", |
|
|
|
doc.getId(), result.totalChunks, stored, result.firstError); |
|
|
|
continue; |
|
|
|
} |
|
|
|
|
|
|
|
// 已入库块保留在库（chunk_count 记实际入库数），缺失区间聚合进 error_message |
|
|
|
markFailed(doc, result.buildSummary(stored)); |
|
|
|
return; |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
/** |
|
|
|
* 分批向量化入库：按 embedBatchSize 逐批 add()，单批失败仅记录缺失区间后继续； |
|
|
|
* 连续失败超过阈值且错误可重试时中止本轮（交给文档级重试），避免打满全部批次 |
|
|
|
* |
|
|
|
* @return 各批结果聚合（成功批已写入 DB） |
|
|
|
*/ |
|
|
|
private BatchResult vectorizeInBatches(String docIdStr, List<Document> annotated, Long docId) { |
|
|
|
BatchResult result = new BatchResult(annotated.size()); |
|
|
|
int consecutiveFailures = 0; |
|
|
|
int batchSize = effectiveBatchSize(); |
|
|
|
|
|
|
|
for (int start = 0; start < annotated.size(); start += batchSize) { |
|
|
|
// 批前复查：文档若在异步处理期间被删除，立即终止后续写入，避免在 vector_store 留下孤儿向量 |
|
|
|
if (docId != null && documentMapper.selectById(docId) == null) { |
|
|
|
log.info("处理期间文档已被删除，终止剩余批次: docId={}", docIdStr); |
|
|
|
result.cancelled = true; |
|
|
|
break; |
|
|
|
} |
|
|
|
|
|
|
|
int end = Math.min(start + batchSize, annotated.size()); |
|
|
|
List<Document> batch = annotated.subList(start, end); |
|
|
|
try { |
|
|
|
pgVectorVectorStore.add(batch); |
|
|
|
consecutiveFailures = 0; |
|
|
|
log.info("批次向量化成功: docId={}, 第{}~{}块, 累计进度 {}/{}", |
|
|
|
docIdStr, start + 1, end, Math.min(end, result.totalChunks), result.totalChunks); |
|
|
|
} catch (Exception e) { |
|
|
|
consecutiveFailures++; |
|
|
|
// 缺失区间：0-based chunkIndex 闭区间 [start, end-1]；错误消息截断防 error_message 过长/回显正文 |
|
|
|
result.recordFailure(start, end - 1, truncate(e.getMessage())); |
|
|
|
log.error("批次向量化失败: docId={}, 第{}~{}块, error={}", |
|
|
|
docIdStr, start + 1, end, truncate(e.getMessage())); |
|
|
|
if (consecutiveFailures >= MAX_CONSECUTIVE_BATCH_FAILURES && isRetryable(e.getMessage())) { |
|
|
|
result.aborted = true; |
|
|
|
// 连续失败中止本轮：把尚未执行的后续批次区间一并记入，避免 error_message 缺失段不完整 |
|
|
|
if (end < annotated.size()) { |
|
|
|
result.recordFailure(end, annotated.size() - 1, "连续失败已中止本轮，后续批次未执行"); |
|
|
|
} |
|
|
|
break; |
|
|
|
} |
|
|
|
} |
|
|
|
} |
|
|
|
return result; |
|
|
|
} |
|
|
|
|
|
|
|
/** |
|
|
|
* 为每个分块写入关联元数据：documentId/chunkIndex/sourceName/title/categoryId/tags/enabled |
|
|
|
*/ |
|
|
|
private List<Document> annotateChunks(List<Document> splitDocuments, DocMeta meta) { |
|
|
|
for (int i = 0; i < splitDocuments.size(); i++) { |
|
|
|
Document d = splitDocuments.get(i); |
|
|
|
Map<String, Object> meta = new HashMap<>(d.getMetadata()); |
|
|
|
meta.put("documentId", String.valueOf(docId)); |
|
|
|
meta.put("chunkIndex", i); |
|
|
|
meta.put("sourceName", doc.getSourceName()); |
|
|
|
meta.put("title", doc.getTitle()); |
|
|
|
if (doc.getCategoryId() != null && doc.getCategoryId() > 0) { |
|
|
|
meta.put("categoryId", String.valueOf(doc.getCategoryId())); |
|
|
|
Map<String, Object> m = new HashMap<>(d.getMetadata()); |
|
|
|
m.put("documentId", String.valueOf(meta.docId())); |
|
|
|
m.put("chunkIndex", i); |
|
|
|
m.put("sourceName", meta.sourceName()); |
|
|
|
m.put("title", meta.title()); |
|
|
|
if (meta.categoryId() != null && meta.categoryId() > 0) { |
|
|
|
m.put("categoryId", String.valueOf(meta.categoryId())); |
|
|
|
} |
|
|
|
if (doc.getTags() != null && doc.getTags().containsKey("tags")) { |
|
|
|
meta.put("tags", doc.getTags().get("tags")); |
|
|
|
if (meta.tagsValue() instanceof List<?> tags && !tags.isEmpty()) { |
|
|
|
m.put("tags", tags); |
|
|
|
} |
|
|
|
// P1-2.1: 标记启用状态，用于 RAG 检索过滤 |
|
|
|
m.put("enabled", String.valueOf(meta.enabled())); |
|
|
|
splitDocuments.set(i, new Document(d.getId(), d.getText(), m)); |
|
|
|
} |
|
|
|
// P1-2.1: 标记启用状态 |
|
|
|
meta.put("enabled", String.valueOf(Boolean.TRUE.equals(doc.getEnabled()))); |
|
|
|
splitDocuments.set(i, new Document(d.getId(), d.getText(), meta)); |
|
|
|
return splitDocuments; |
|
|
|
} |
|
|
|
|
|
|
|
List<Document> enrichedDocuments = myKeywordEnricher.enrichDocuments(splitDocuments); |
|
|
|
pgVectorVectorStore.add(enrichedDocuments); |
|
|
|
// ==================== 私有工具 ==================== |
|
|
|
|
|
|
|
doc.setStatus("READY"); |
|
|
|
doc.setChunkCount(enrichedDocuments.size()); |
|
|
|
doc.setErrorMessage(null); |
|
|
|
documentMapper.updateById(doc); |
|
|
|
/** 仅 patch 状态/分块数/错误信息，避免覆盖用户在异步期间的并发编辑（如改标题/分类/toggle enabled） */ |
|
|
|
private void patchStatus(Long docId, String status, Integer chunkCount, String errorMessage) { |
|
|
|
documentMapper.update(null, new LambdaUpdateWrapper<KnowledgeDocument>() |
|
|
|
.eq(KnowledgeDocument::getId, docId) |
|
|
|
.set(KnowledgeDocument::getStatus, status) |
|
|
|
.set(KnowledgeDocument::getChunkCount, chunkCount) |
|
|
|
.set(KnowledgeDocument::getErrorMessage, errorMessage)); |
|
|
|
} |
|
|
|
|
|
|
|
log.info("异步重新处理文档成功: id={}, title={}, chunks={}", docId, doc.getTitle(), enrichedDocuments.size()); |
|
|
|
private void markFailed(KnowledgeDocument doc, String message) { |
|
|
|
String docIdStr = String.valueOf(doc.getId()); |
|
|
|
patchStatus(doc.getId(), "FAILED", safeCountVectors(docIdStr), message); |
|
|
|
log.error("文档向量化失败: id={}, title={}, error={}", doc.getId(), doc.getTitle(), message); |
|
|
|
} |
|
|
|
|
|
|
|
/** 统计实际已入库向量数（失败时容错，DB 抖动不阻断状态收敛） */ |
|
|
|
private int safeCountVectors(String docIdStr) { |
|
|
|
try { |
|
|
|
return countStoredVectors(docIdStr); |
|
|
|
} catch (Exception e) { |
|
|
|
doc.setStatus("FAILED"); |
|
|
|
doc.setErrorMessage(e.getMessage()); |
|
|
|
documentMapper.updateById(doc); |
|
|
|
log.error("异步重新处理文档失败: id={}, title={}", docId, doc.getTitle(), e); |
|
|
|
log.warn("统计已入库向量数失败: docId={}, error={}", docIdStr, e.getMessage()); |
|
|
|
return 0; |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
/** 截断错误消息，避免 error_message 过长或回显大段文档文本 */ |
|
|
|
private static String truncate(String msg) { |
|
|
|
if (msg == null) { |
|
|
|
return null; |
|
|
|
} |
|
|
|
return msg.length() <= 500 ? msg : msg.substring(0, 500) + "…(已截断)"; |
|
|
|
} |
|
|
|
|
|
|
|
/** 实际已入库向量条数：以 DB 为准，天然吸收"失败批内偶发的部分写入" */ |
|
|
|
private int countStoredVectors(String docIdStr) { |
|
|
|
Integer c = jdbcTemplate.queryForObject( |
|
|
|
"SELECT count(*) FROM vector_store WHERE metadata->>'documentId' = ?", |
|
|
|
Integer.class, docIdStr); |
|
|
|
return c == null ? 0 : c; |
|
|
|
} |
|
|
|
|
|
|
|
/** 物理删除该文档在 vector_store 的全部向量（vector_store 无逻辑删除语义） */ |
|
|
|
private void deleteVectorsByDocumentId(String docIdStr) { |
|
|
|
List<String> ids = jdbcTemplate.queryForList( |
|
|
|
"SELECT id::text FROM vector_store WHERE metadata->>'documentId' = ?", |
|
|
|
String.class, docIdStr); |
|
|
|
if (!ids.isEmpty()) { |
|
|
|
pgVectorVectorStore.delete(ids); |
|
|
|
log.debug("清理旧向量: documentId={}, count={}", docIdStr, ids.size()); |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
private int effectiveBatchSize() { |
|
|
|
return embedBatchSize > 0 ? embedBatchSize : 50; |
|
|
|
} |
|
|
|
|
|
|
|
/** 简单指数退避等待；被中断时复位中断标志并返回 false（放弃处理） */ |
|
|
|
private static boolean sleepBackoff(int attempt) { |
|
|
|
long ms = RETRY_BASE_DELAY_MS * (1L << (attempt - 1)); |
|
|
|
try { |
|
|
|
Thread.sleep(ms); |
|
|
|
return true; |
|
|
|
} catch (InterruptedException e) { |
|
|
|
Thread.currentThread().interrupt(); |
|
|
|
return false; |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
/** 仅对疑似瞬时/基础设施类错误做文档级重试，4xx 参数类错误不空转 */ |
|
|
|
private static boolean isRetryable(String msg) { |
|
|
|
// 消息缺失的未知错误不做整篇重试（避免对 NPE 等非瞬时错误空转 2 次），仅对明确瞬时/限流特征重试 |
|
|
|
if (msg == null || msg.isBlank()) { |
|
|
|
return false; |
|
|
|
} |
|
|
|
String m = msg.toLowerCase(); |
|
|
|
return m.contains("timeout") || m.contains("timed out") || m.contains("connection") |
|
|
|
|| m.contains("429") || m.contains("too many request") || m.contains("rate limit") |
|
|
|
|| m.contains(" 500") || m.contains(" 502") || m.contains(" 503") || m.contains(" 504") |
|
|
|
|| m.contains("socket") || m.contains("i/o error") || m.contains("internal server error") |
|
|
|
|| m.contains("service unavailable") || m.contains("bad gateway"); |
|
|
|
} |
|
|
|
|
|
|
|
/** |
|
|
|
@ -206,4 +393,73 @@ public class DocumentProcessingService { |
|
|
|
} |
|
|
|
return null; |
|
|
|
} |
|
|
|
|
|
|
|
/** |
|
|
|
* 分批向量化结果聚合：成功批已入库；失败批记录缺失 chunkIndex 区间与首个错误 |
|
|
|
*/ |
|
|
|
private static final class BatchResult { |
|
|
|
final int totalChunks; |
|
|
|
/** 每项为 [start, end]（0-based chunkIndex 闭区间），最多保留前 8 段 */ |
|
|
|
final List<int[]> failedRanges = new ArrayList<>(); |
|
|
|
String firstError; |
|
|
|
boolean aborted; |
|
|
|
/** 处理期间文档被删除时置位：调用方应终止且不再写状态 */ |
|
|
|
boolean cancelled; |
|
|
|
|
|
|
|
BatchResult(int totalChunks) { |
|
|
|
this.totalChunks = totalChunks; |
|
|
|
} |
|
|
|
|
|
|
|
void recordFailure(int start, int end, String error) { |
|
|
|
failedRanges.add(new int[]{start, end}); |
|
|
|
if (firstError == null) { |
|
|
|
firstError = error; |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
boolean hasFailure() { |
|
|
|
return !failedRanges.isEmpty(); |
|
|
|
} |
|
|
|
|
|
|
|
/** 例：已入库 320/400 块，缺失第 321~400 块 向量化失败: <原因> */ |
|
|
|
String buildSummary(int storedChunks) { |
|
|
|
String ranges = failedRanges.stream().limit(8) |
|
|
|
.map(r -> r[0] == r[1] |
|
|
|
? "第 " + (r[0] + 1) + " 块" |
|
|
|
: "第 " + (r[0] + 1) + "~" + (r[1] + 1) + " 块") |
|
|
|
.collect(Collectors.joining("、")); |
|
|
|
if (failedRanges.size() > 8) { |
|
|
|
ranges += "…共 " + failedRanges.size() + " 段"; |
|
|
|
} |
|
|
|
String msg = "已入库 " + storedChunks + "/" + totalChunks + " 块，缺失 " + ranges + " 向量化失败"; |
|
|
|
if (aborted) { |
|
|
|
msg += "（连续失败已中止本轮）"; |
|
|
|
} |
|
|
|
if (firstError != null) { |
|
|
|
msg += ": " + firstError; |
|
|
|
} |
|
|
|
return msg; |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
/** |
|
|
|
* 分块元数据值对象：统一两个异步入口（新上传/reprocess）的 metadata 标注逻辑 |
|
|
|
*/ |
|
|
|
private record DocMeta(Long docId, String sourceName, String title, Long categoryId, |
|
|
|
Object tagsValue, boolean enabled) { |
|
|
|
|
|
|
|
static DocMeta of(Long docId, String sourceName, String title, Long categoryId, |
|
|
|
List<String> tags, boolean enabled) { |
|
|
|
return new DocMeta(docId, sourceName, title, categoryId, tags, enabled); |
|
|
|
} |
|
|
|
|
|
|
|
static DocMeta fromDocument(KnowledgeDocument doc) { |
|
|
|
Object tagsValue = null; |
|
|
|
if (doc.getTags() != null && doc.getTags().containsKey("tags")) { |
|
|
|
tagsValue = doc.getTags().get("tags"); |
|
|
|
} |
|
|
|
return new DocMeta(doc.getId(), doc.getSourceName(), doc.getTitle(), |
|
|
|
doc.getCategoryId(), tagsValue, Boolean.TRUE.equals(doc.getEnabled())); |
|
|
|
} |
|
|
|
} |
|
|
|
} |
