Browse Source
知识库文档管理 P0 阶段功能完善与体验优化
知识库文档管理 P0 阶段功能完善与体验优化
- 新增文档关键词搜索（title + source_name LIKE 查询，前端 300ms 防抖）
- 文档列表增加文件大小（formatBytes）和分类名称列
- 分类管理支持编辑弹窗、树形层级展示（最多3级）、创建时选择父分类
- 新建 DocumentProcessingService 异步处理服务，上传/重处理改为同步插入
PROCESSING 记录 + 异步执行，前端自动轮询状态
- 修复分页参数越界、LIKE 通配符未转义、异步删除竞态条件等安全问题
- 修复全选 checkbox 始终选中、分类排序方向不一致、文本上传缺少认证头等缺陷
- 移除 DocUpload.js 遗留调试日志
dev-mcp
7 changed files with 613 additions and 161 deletions
-
22src/main/java/com/wok/supportbot/controller/DocumentController.java
-
194src/main/java/com/wok/supportbot/service/DocumentProcessingService.java
-
120src/main/java/com/wok/supportbot/service/DocumentService.java
-
300src/main/resources/static/components/CategoryManager.js
-
102src/main/resources/static/components/DocList.js
-
27src/main/resources/static/components/DocUpload.js
-
7src/main/resources/static/js/api.js
@ -0,0 +1,194 @@ |
|||
package com.wok.supportbot.service; |
|||
|
|||
import com.wok.supportbot.dao.KnowledgeDocumentMapper; |
|||
import com.wok.supportbot.document.transform.MyKeywordEnricher; |
|||
import com.wok.supportbot.document.transform.MyTokenTextSplitter; |
|||
import com.wok.supportbot.entity.KnowledgeDocument; |
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.springframework.ai.document.Document; |
|||
import org.springframework.ai.vectorstore.VectorStore; |
|||
import org.springframework.beans.factory.annotation.Autowired; |
|||
import org.springframework.jdbc.core.JdbcTemplate; |
|||
import org.springframework.scheduling.annotation.Async; |
|||
import org.springframework.stereotype.Service; |
|||
import org.springframework.transaction.annotation.Propagation; |
|||
import org.springframework.transaction.annotation.Transactional; |
|||
|
|||
import java.util.HashMap; |
|||
import java.util.List; |
|||
import java.util.Map; |
|||
|
|||
/** |
|||
* 文档异步处理服务 |
|||
* 负责文档的分块、关键词提取、向量化等耗时操作，在后台线程中执行 |
|||
*/ |
|||
@Service |
|||
@Slf4j |
|||
public class DocumentProcessingService { |
|||
|
|||
@Autowired |
|||
private KnowledgeDocumentMapper documentMapper; |
|||
|
|||
@Autowired |
|||
private MyTokenTextSplitter myTokenTextSplitter; |
|||
|
|||
@Autowired |
|||
private MyKeywordEnricher myKeywordEnricher; |
|||
|
|||
@Autowired |
|||
private VectorStore pgVectorVectorStore; |
|||
|
|||
@Autowired |
|||
private JdbcTemplate jdbcTemplate; |
|||
|
|||
/** |
|||
* 异步处理文档：分块 → 关键词提取 → 向量化 → 更新状态 |
|||
* 使用 REQUIRES_NEW 在独立事务中执行，避免与主事务冲突 |
|||
* |
|||
* @param docId 文档ID |
|||
* @param documents 已解析的原始文档列表 |
|||
* @param sourceName 源文件名 |
|||
* @param title 文档标题 |
|||
* @param categoryId 分类ID |
|||
* @param tags 标签列表 |
|||
*/ |
|||
@Async |
|||
@Transactional(propagation = Propagation.REQUIRES_NEW, rollbackFor = Exception.class) |
|||
public void processDocumentAsync(Long docId, List<Document> documents, String sourceName, |
|||
String title, Long categoryId, List<String> tags) { |
|||
// 等待主事务提交，确保文档记录可见（最多重试 5 次，每次 200ms） |
|||
KnowledgeDocument doc = waitForDocument(docId); |
|||
if (doc == null) { |
|||
log.error("异步处理文档失败：文档不存在（等待超时）, id={}", docId); |
|||
return; |
|||
} |
|||
|
|||
try { |
|||
// 1. 分块处理 |
|||
List<Document> splitDocuments = myTokenTextSplitter.splitDocuments(documents); |
|||
|
|||
// 2. 为每个分块设置 metadata |
|||
for (int i = 0; i < splitDocuments.size(); i++) { |
|||
Document d = splitDocuments.get(i); |
|||
Map<String, Object> meta = new HashMap<>(d.getMetadata()); |
|||
meta.put("documentId", String.valueOf(docId)); |
|||
meta.put("chunkIndex", i); |
|||
meta.put("sourceName", sourceName); |
|||
meta.put("title", title); |
|||
if (categoryId != null && categoryId > 0) { |
|||
meta.put("categoryId", String.valueOf(categoryId)); |
|||
} |
|||
if (tags != null && !tags.isEmpty()) { |
|||
meta.put("tags", tags); |
|||
} |
|||
splitDocuments.set(i, new Document(d.getId(), d.getText(), meta)); |
|||
} |
|||
|
|||
// 3. 关键词提取 |
|||
List<Document> enrichedDocuments = myKeywordEnricher.enrichDocuments(splitDocuments); |
|||
|
|||
// 4. 向量化存储 |
|||
pgVectorVectorStore.add(enrichedDocuments); |
|||
|
|||
// 5. 更新状态为 READY |
|||
doc.setStatus("READY"); |
|||
doc.setChunkCount(enrichedDocuments.size()); |
|||
documentMapper.updateById(doc); |
|||
|
|||
log.info("异步处理文档完成: id={}, title={}, chunks={}", docId, title, enrichedDocuments.size()); |
|||
|
|||
} catch (Exception e) { |
|||
doc.setStatus("FAILED"); |
|||
doc.setErrorMessage(e.getMessage()); |
|||
documentMapper.updateById(doc); |
|||
log.error("异步处理文档失败: id={}, title={}", docId, title, e); |
|||
} |
|||
} |
|||
|
|||
/** |
|||
* 异步重新处理文档（重新分块 + 向量化） |
|||
* |
|||
* @param docId 文档ID |
|||
* @param documents 解析后的文档列表 |
|||
*/ |
|||
@Async |
|||
@Transactional(propagation = Propagation.REQUIRES_NEW, rollbackFor = Exception.class) |
|||
public void reprocessDocumentAsync(Long docId, List<Document> documents) { |
|||
// 等待主事务提交，确保文档记录可见 |
|||
KnowledgeDocument doc = waitForDocument(docId); |
|||
if (doc == null) { |
|||
log.error("异步重新处理文档失败：文档不存在（等待超时）, id={}", docId); |
|||
return; |
|||
} |
|||
|
|||
try { |
|||
// 删除旧向量 |
|||
String sql = "SELECT id::text FROM vector_store WHERE metadata->>'documentId' = ?"; |
|||
List<String> oldIds = jdbcTemplate.queryForList(sql, String.class, String.valueOf(docId)); |
|||
if (!oldIds.isEmpty()) { |
|||
pgVectorVectorStore.delete(oldIds); |
|||
} |
|||
|
|||
// 重新分块 |
|||
List<Document> splitDocuments = myTokenTextSplitter.splitDocuments(documents); |
|||
|
|||
for (int i = 0; i < splitDocuments.size(); i++) { |
|||
Document d = splitDocuments.get(i); |
|||
Map<String, Object> meta = new HashMap<>(d.getMetadata()); |
|||
meta.put("documentId", String.valueOf(docId)); |
|||
meta.put("chunkIndex", i); |
|||
meta.put("sourceName", doc.getSourceName()); |
|||
meta.put("title", doc.getTitle()); |
|||
if (doc.getCategoryId() != null && doc.getCategoryId() > 0) { |
|||
meta.put("categoryId", String.valueOf(doc.getCategoryId())); |
|||
} |
|||
if (doc.getTags() != null && doc.getTags().containsKey("tags")) { |
|||
meta.put("tags", doc.getTags().get("tags")); |
|||
} |
|||
splitDocuments.set(i, new Document(d.getId(), d.getText(), meta)); |
|||
} |
|||
|
|||
List<Document> enrichedDocuments = myKeywordEnricher.enrichDocuments(splitDocuments); |
|||
pgVectorVectorStore.add(enrichedDocuments); |
|||
|
|||
doc.setStatus("READY"); |
|||
doc.setChunkCount(enrichedDocuments.size()); |
|||
doc.setErrorMessage(null); |
|||
documentMapper.updateById(doc); |
|||
|
|||
log.info("异步重新处理文档成功: id={}, title={}, chunks={}", docId, doc.getTitle(), enrichedDocuments.size()); |
|||
|
|||
} catch (Exception e) { |
|||
doc.setStatus("FAILED"); |
|||
doc.setErrorMessage(e.getMessage()); |
|||
documentMapper.updateById(doc); |
|||
log.error("异步重新处理文档失败: id={}, title={}", docId, doc.getTitle(), e); |
|||
} |
|||
} |
|||
|
|||
/** |
|||
* 等待主事务提交后文档可见（最多重试 5 次，每次 200ms，总计 1 秒） |
|||
* 解决 @Async 任务可能在主事务提交前启动的竞态条件 |
|||
* 同时检查文档是否已被删除（用户可能在异步处理前删除了文档） |
|||
*/ |
|||
private KnowledgeDocument waitForDocument(Long docId) { |
|||
for (int i = 0; i < 5; i++) { |
|||
KnowledgeDocument doc = documentMapper.selectById(docId); |
|||
if (doc != null) { |
|||
// 检查是否已被逻辑删除（用户可能在处理前删除了文档） |
|||
if (doc.isDelete()) { |
|||
log.info("文档已被删除，跳过异步处理: id={}", docId); |
|||
return null; |
|||
} |
|||
return doc; |
|||
} |
|||
try { |
|||
Thread.sleep(200); |
|||
} catch (InterruptedException e) { |
|||
Thread.currentThread().interrupt(); |
|||
return null; |
|||
} |
|||
} |
|||
return null; |
|||
} |
|||
} |
|||
Write
Preview
Loading…
Cancel
Save
Reference in new issue