Browse Source
知识库文档管理 P0 阶段功能完善与体验优化
知识库文档管理 P0 阶段功能完善与体验优化
- 新增文档关键词搜索（title + source_name LIKE 查询，前端 300ms 防抖）
- 文档列表增加文件大小（formatBytes）和分类名称列
- 分类管理支持编辑弹窗、树形层级展示（最多3级）、创建时选择父分类
- 新建 DocumentProcessingService 异步处理服务，上传/重处理改为同步插入
PROCESSING 记录 + 异步执行，前端自动轮询状态
- 修复分页参数越界、LIKE 通配符未转义、异步删除竞态条件等安全问题
- 修复全选 checkbox 始终选中、分类排序方向不一致、文本上传缺少认证头等缺陷
- 移除 DocUpload.js 遗留调试日志
dev-mcp
7 changed files with 613 additions and 161 deletions
-
22src/main/java/com/wok/supportbot/controller/DocumentController.java
-
194src/main/java/com/wok/supportbot/service/DocumentProcessingService.java
-
120src/main/java/com/wok/supportbot/service/DocumentService.java
-
300src/main/resources/static/components/CategoryManager.js
-
102src/main/resources/static/components/DocList.js
-
29src/main/resources/static/components/DocUpload.js
-
7src/main/resources/static/js/api.js
@ -0,0 +1,194 @@ |
|||||
|
package com.wok.supportbot.service; |
||||
|
|
||||
|
import com.wok.supportbot.dao.KnowledgeDocumentMapper; |
||||
|
import com.wok.supportbot.document.transform.MyKeywordEnricher; |
||||
|
import com.wok.supportbot.document.transform.MyTokenTextSplitter; |
||||
|
import com.wok.supportbot.entity.KnowledgeDocument; |
||||
|
import lombok.extern.slf4j.Slf4j; |
||||
|
import org.springframework.ai.document.Document; |
||||
|
import org.springframework.ai.vectorstore.VectorStore; |
||||
|
import org.springframework.beans.factory.annotation.Autowired; |
||||
|
import org.springframework.jdbc.core.JdbcTemplate; |
||||
|
import org.springframework.scheduling.annotation.Async; |
||||
|
import org.springframework.stereotype.Service; |
||||
|
import org.springframework.transaction.annotation.Propagation; |
||||
|
import org.springframework.transaction.annotation.Transactional; |
||||
|
|
||||
|
import java.util.HashMap; |
||||
|
import java.util.List; |
||||
|
import java.util.Map; |
||||
|
|
||||
|
/** |
||||
|
* 文档异步处理服务 |
||||
|
* 负责文档的分块、关键词提取、向量化等耗时操作，在后台线程中执行 |
||||
|
*/ |
||||
|
@Service |
||||
|
@Slf4j |
||||
|
public class DocumentProcessingService { |
||||
|
|
||||
|
@Autowired |
||||
|
private KnowledgeDocumentMapper documentMapper; |
||||
|
|
||||
|
@Autowired |
||||
|
private MyTokenTextSplitter myTokenTextSplitter; |
||||
|
|
||||
|
@Autowired |
||||
|
private MyKeywordEnricher myKeywordEnricher; |
||||
|
|
||||
|
@Autowired |
||||
|
private VectorStore pgVectorVectorStore; |
||||
|
|
||||
|
@Autowired |
||||
|
private JdbcTemplate jdbcTemplate; |
||||
|
|
||||
|
/** |
||||
|
* 异步处理文档：分块 → 关键词提取 → 向量化 → 更新状态 |
||||
|
* 使用 REQUIRES_NEW 在独立事务中执行，避免与主事务冲突 |
||||
|
* |
||||
|
* @param docId 文档ID |
||||
|
* @param documents 已解析的原始文档列表 |
||||
|
* @param sourceName 源文件名 |
||||
|
* @param title 文档标题 |
||||
|
* @param categoryId 分类ID |
||||
|
* @param tags 标签列表 |
||||
|
*/ |
||||
|
@Async |
||||
|
@Transactional(propagation = Propagation.REQUIRES_NEW, rollbackFor = Exception.class) |
||||
|
public void processDocumentAsync(Long docId, List<Document> documents, String sourceName, |
||||
|
String title, Long categoryId, List<String> tags) { |
||||
|
// 等待主事务提交，确保文档记录可见（最多重试 5 次，每次 200ms） |
||||
|
KnowledgeDocument doc = waitForDocument(docId); |
||||
|
if (doc == null) { |
||||
|
log.error("异步处理文档失败：文档不存在（等待超时）, id={}", docId); |
||||
|
return; |
||||
|
} |
||||
|
|
||||
|
try { |
||||
|
// 1. 分块处理 |
||||
|
List<Document> splitDocuments = myTokenTextSplitter.splitDocuments(documents); |
||||
|
|
||||
|
// 2. 为每个分块设置 metadata |
||||
|
for (int i = 0; i < splitDocuments.size(); i++) { |
||||
|
Document d = splitDocuments.get(i); |
||||
|
Map<String, Object> meta = new HashMap<>(d.getMetadata()); |
||||
|
meta.put("documentId", String.valueOf(docId)); |
||||
|
meta.put("chunkIndex", i); |
||||
|
meta.put("sourceName", sourceName); |
||||
|
meta.put("title", title); |
||||
|
if (categoryId != null && categoryId > 0) { |
||||
|
meta.put("categoryId", String.valueOf(categoryId)); |
||||
|
} |
||||
|
if (tags != null && !tags.isEmpty()) { |
||||
|
meta.put("tags", tags); |
||||
|
} |
||||
|
splitDocuments.set(i, new Document(d.getId(), d.getText(), meta)); |
||||
|
} |
||||
|
|
||||
|
// 3. 关键词提取 |
||||
|
List<Document> enrichedDocuments = myKeywordEnricher.enrichDocuments(splitDocuments); |
||||
|
|
||||
|
// 4. 向量化存储 |
||||
|
pgVectorVectorStore.add(enrichedDocuments); |
||||
|
|
||||
|
// 5. 更新状态为 READY |
||||
|
doc.setStatus("READY"); |
||||
|
doc.setChunkCount(enrichedDocuments.size()); |
||||
|
documentMapper.updateById(doc); |
||||
|
|
||||
|
log.info("异步处理文档完成: id={}, title={}, chunks={}", docId, title, enrichedDocuments.size()); |
||||
|
|
||||
|
} catch (Exception e) { |
||||
|
doc.setStatus("FAILED"); |
||||
|
doc.setErrorMessage(e.getMessage()); |
||||
|
documentMapper.updateById(doc); |
||||
|
log.error("异步处理文档失败: id={}, title={}", docId, title, e); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
/** |
||||
|
* 异步重新处理文档（重新分块 + 向量化） |
||||
|
* |
||||
|
* @param docId 文档ID |
||||
|
* @param documents 解析后的文档列表 |
||||
|
*/ |
||||
|
@Async |
||||
|
@Transactional(propagation = Propagation.REQUIRES_NEW, rollbackFor = Exception.class) |
||||
|
public void reprocessDocumentAsync(Long docId, List<Document> documents) { |
||||
|
// 等待主事务提交，确保文档记录可见 |
||||
|
KnowledgeDocument doc = waitForDocument(docId); |
||||
|
if (doc == null) { |
||||
|
log.error("异步重新处理文档失败：文档不存在（等待超时）, id={}", docId); |
||||
|
return; |
||||
|
} |
||||
|
|
||||
|
try { |
||||
|
// 删除旧向量 |
||||
|
String sql = "SELECT id::text FROM vector_store WHERE metadata->>'documentId' = ?"; |
||||
|
List<String> oldIds = jdbcTemplate.queryForList(sql, String.class, String.valueOf(docId)); |
||||
|
if (!oldIds.isEmpty()) { |
||||
|
pgVectorVectorStore.delete(oldIds); |
||||
|
} |
||||
|
|
||||
|
// 重新分块 |
||||
|
List<Document> splitDocuments = myTokenTextSplitter.splitDocuments(documents); |
||||
|
|
||||
|
for (int i = 0; i < splitDocuments.size(); i++) { |
||||
|
Document d = splitDocuments.get(i); |
||||
|
Map<String, Object> meta = new HashMap<>(d.getMetadata()); |
||||
|
meta.put("documentId", String.valueOf(docId)); |
||||
|
meta.put("chunkIndex", i); |
||||
|
meta.put("sourceName", doc.getSourceName()); |
||||
|
meta.put("title", doc.getTitle()); |
||||
|
if (doc.getCategoryId() != null && doc.getCategoryId() > 0) { |
||||
|
meta.put("categoryId", String.valueOf(doc.getCategoryId())); |
||||
|
} |
||||
|
if (doc.getTags() != null && doc.getTags().containsKey("tags")) { |
||||
|
meta.put("tags", doc.getTags().get("tags")); |
||||
|
} |
||||
|
splitDocuments.set(i, new Document(d.getId(), d.getText(), meta)); |
||||
|
} |
||||
|
|
||||
|
List<Document> enrichedDocuments = myKeywordEnricher.enrichDocuments(splitDocuments); |
||||
|
pgVectorVectorStore.add(enrichedDocuments); |
||||
|
|
||||
|
doc.setStatus("READY"); |
||||
|
doc.setChunkCount(enrichedDocuments.size()); |
||||
|
doc.setErrorMessage(null); |
||||
|
documentMapper.updateById(doc); |
||||
|
|
||||
|
log.info("异步重新处理文档成功: id={}, title={}, chunks={}", docId, doc.getTitle(), enrichedDocuments.size()); |
||||
|
|
||||
|
} catch (Exception e) { |
||||
|
doc.setStatus("FAILED"); |
||||
|
doc.setErrorMessage(e.getMessage()); |
||||
|
documentMapper.updateById(doc); |
||||
|
log.error("异步重新处理文档失败: id={}, title={}", docId, doc.getTitle(), e); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
/** |
||||
|
* 等待主事务提交后文档可见（最多重试 5 次，每次 200ms，总计 1 秒） |
||||
|
* 解决 @Async 任务可能在主事务提交前启动的竞态条件 |
||||
|
* 同时检查文档是否已被删除（用户可能在异步处理前删除了文档） |
||||
|
*/ |
||||
|
private KnowledgeDocument waitForDocument(Long docId) { |
||||
|
for (int i = 0; i < 5; i++) { |
||||
|
KnowledgeDocument doc = documentMapper.selectById(docId); |
||||
|
if (doc != null) { |
||||
|
// 检查是否已被逻辑删除（用户可能在处理前删除了文档） |
||||
|
if (doc.isDelete()) { |
||||
|
log.info("文档已被删除，跳过异步处理: id={}", docId); |
||||
|
return null; |
||||
|
} |
||||
|
return doc; |
||||
|
} |
||||
|
try { |
||||
|
Thread.sleep(200); |
||||
|
} catch (InterruptedException e) { |
||||
|
Thread.currentThread().interrupt(); |
||||
|
return null; |
||||
|
} |
||||
|
} |
||||
|
return null; |
||||
|
} |
||||
|
} |
||||
Write
Preview
Loading…
Cancel
Save
Reference in new issue