You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
1338 lines
54 KiB
1338 lines
54 KiB
package com.wok.supportbot.service;
|
|
|
|
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
|
|
import com.wok.supportbot.dao.KnowledgeCategoryMapper;
|
|
import com.wok.supportbot.dao.KnowledgeDocumentMapper;
|
|
import com.wok.supportbot.document.extract.JsonDocumentLoader;
|
|
import com.wok.supportbot.document.extract.MarkdownDocumentLoader;
|
|
import com.wok.supportbot.document.extract.SimpleStringDocumentReader;
|
|
import com.wok.supportbot.document.extract.TikaDocumentReader;
|
|
import com.wok.supportbot.document.transform.MyTokenTextSplitter;
|
|
import com.wok.supportbot.entity.CategoryNode;
|
|
import com.wok.supportbot.entity.KnowledgeCategory;
|
|
import com.wok.supportbot.entity.KnowledgeDocument;
|
|
import com.wok.supportbot.entity.KnowledgeFolder;
|
|
import com.wok.supportbot.entity.SearchResult;
|
|
import lombok.extern.slf4j.Slf4j;
|
|
import org.springframework.ai.document.Document;
|
|
import org.springframework.ai.vectorstore.SearchRequest;
|
|
import org.springframework.ai.vectorstore.VectorStore;
|
|
import org.springframework.ai.vectorstore.filter.FilterExpressionBuilder;
|
|
import org.springframework.beans.factory.annotation.Autowired;
|
|
import org.springframework.jdbc.core.JdbcTemplate;
|
|
import org.springframework.stereotype.Service;
|
|
import org.springframework.transaction.annotation.Transactional;
|
|
import org.springframework.web.multipart.MultipartFile;
|
|
|
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
|
|
|
import java.nio.charset.StandardCharsets;
|
|
import java.security.MessageDigest;
|
|
import java.security.NoSuchAlgorithmException;
|
|
import java.util.*;
|
|
import java.util.stream.Collectors;
|
|
|
|
/**
|
|
* 知识库文档管理服务
|
|
* 统一管理文档的上传、删除、搜索、统计、分类等操作
|
|
*/
|
|
@Service
|
|
@Slf4j
|
|
public class DocumentService {
|
|
|
|
@Autowired
|
|
private KnowledgeDocumentMapper documentMapper;
|
|
|
|
@Autowired
|
|
private KnowledgeCategoryMapper categoryMapper;
|
|
|
|
@Autowired
|
|
private JdbcTemplate jdbcTemplate;
|
|
|
|
@Autowired
|
|
private VectorStore pgVectorVectorStore;
|
|
|
|
@Autowired
|
|
private MyTokenTextSplitter myTokenTextSplitter;
|
|
|
|
@Autowired
|
|
private TikaDocumentReader tikaDocumentReader;
|
|
|
|
@Autowired
|
|
private SimpleStringDocumentReader simpleStringDocumentReader;
|
|
|
|
@Autowired
|
|
private MarkdownDocumentLoader markdownDocumentLoader;
|
|
|
|
@Autowired
|
|
private JsonDocumentLoader jsonDocumentLoader;
|
|
|
|
@Autowired
|
|
private DocumentProcessingService documentProcessingService;
|
|
|
|
@Autowired
|
|
private FolderService folderService;
|
|
|
|
@Autowired
|
|
private com.wok.supportbot.config.FileStorageConfig fileStorageConfig;
|
|
|
|
@Autowired
|
|
private com.wok.supportbot.rag.CategoryFilter categoryFilter;
|
|
|
|
// ==================== 文档上传 ====================
|
|
|
|
/**
|
|
* 统一文档上传流程：创建记录（PROCESSING）→ 触发异步处理
|
|
* 同步阶段：内容去重检查 + 创建 PROCESSING 记录
|
|
* 异步阶段：分块 → 关键词提取 → 向量化 → 更新状态为 READY
|
|
*
|
|
* @param documents 解析后的文档列表
|
|
* @param title 文档标题
|
|
* @param sourceName 源文件名
|
|
* @param fileType 文件类型
|
|
* @param fileSize 文件大小
|
|
* @param content 原文内容（截断预览）
|
|
* @param categoryId 分类ID
|
|
* @param tags 标签列表
|
|
* @param chunkSize 分块大小（可选，覆盖全局配置）
|
|
* @param overlap 重叠大小（可选，覆盖全局配置）
|
|
* @param filePath 原始文件存储路径（可选，null表示无原始文件）
|
|
* @param folderId 目录ID（可选，null/0表示不指定目录）
|
|
* @return 创建完成的文档记录（status=PROCESSING）
|
|
*/
|
|
@Transactional(rollbackFor = Exception.class)
|
|
public KnowledgeDocument uploadDocument(List<Document> documents, String title, String sourceName,
|
|
String fileType, Long fileSize, String content,
|
|
Long categoryId, List<String> tags,
|
|
Integer chunkSize, Integer overlap,
|
|
String filePath, Long folderId) {
|
|
// 0. 若指定目录，校验目录存在并强制使用目录所属分类
|
|
if (folderId != null && folderId > 0) {
|
|
KnowledgeFolder folder = folderService.getFolderById(folderId);
|
|
if (folder == null) {
|
|
throw new RuntimeException("目录不存在");
|
|
}
|
|
categoryId = folder.getCategoryId();
|
|
}
|
|
|
|
// 1. 内容去重检查
|
|
String contentHash = computeContentHash(content);
|
|
if (contentHash != null) {
|
|
String duplicateTitle = checkContentDuplicate(contentHash, categoryId, folderId);
|
|
if (duplicateTitle != null) {
|
|
throw new RuntimeException("文档内容重复，该目录下已有相同内容的文档: " + duplicateTitle);
|
|
}
|
|
}
|
|
|
|
// 存储 per-doc 分块参数到 extraConfig（便于重新处理时复用）
|
|
Map<String, Object> extraConfig = new HashMap<>();
|
|
if (chunkSize != null) extraConfig.put("chunkSize", chunkSize);
|
|
if (overlap != null) extraConfig.put("overlap", overlap);
|
|
|
|
// 2. 创建文档记录（状态 PROCESSING）
|
|
KnowledgeDocument docRecord = KnowledgeDocument.builder()
|
|
.title(title != null ? title : sourceName)
|
|
.sourceName(sourceName)
|
|
.fileType(fileType)
|
|
.fileSize(fileSize != null ? fileSize : 0L)
|
|
.filePath(filePath)
|
|
.content(content != null && content.length() > 2000 ? content.substring(0, 2000) : content)
|
|
.categoryId(categoryId != null ? categoryId : 0L)
|
|
.folderId(folderId != null ? folderId : 0L)
|
|
.tags(tags != null ? Map.of("tags", tags) : null)
|
|
.contentHash(contentHash)
|
|
.enabled(true)
|
|
.extraConfig(extraConfig.isEmpty() ? null : extraConfig)
|
|
.status("PROCESSING")
|
|
.chunkCount(0)
|
|
.build();
|
|
documentMapper.insert(docRecord);
|
|
|
|
// 3. 触发异步处理（分块 → 关键词 → 向量化 → 更新状态）
|
|
documentProcessingService.processDocumentAsync(
|
|
docRecord.getId(), documents, sourceName, title, categoryId, tags,
|
|
chunkSize, overlap);
|
|
|
|
log.info("文档已创建，后台处理中: id={}, title={}", docRecord.getId(), docRecord.getTitle());
|
|
return docRecord;
|
|
}
|
|
|
|
/**
|
|
* 解析文件并上传（同时保存原始文件到本地）
|
|
*/
|
|
public KnowledgeDocument uploadFile(MultipartFile file, String title, Long categoryId, Long folderId,
|
|
List<String> tags, Integer chunkSize, Integer overlap) {
|
|
// 1. 先保存原始文件到本地磁盘（必须在解析之前，因为 transferTo 只能调用一次）
|
|
String relativePath = saveFileToLocal(file);
|
|
java.io.File savedFile = fileStorageConfig.getFilePath(relativePath).toFile();
|
|
|
|
// 2. 使用保存的文件进行解析（避免 MultipartFile 的临时文件问题）
|
|
List<Document> documents = tikaDocumentReader.readFromFile(savedFile);
|
|
// 剥离路径前缀：文件夹上传时浏览器 filename 携带相对路径（含根目录名），否则标题会带「目录/」前缀
|
|
String sourceName = stripPath(file.getOriginalFilename());
|
|
String fileType = getFileExtension(sourceName);
|
|
|
|
// 3. 创建文档记录并保存文件路径
|
|
return uploadDocument(documents,
|
|
title != null ? title : sourceName,
|
|
sourceName,
|
|
fileType,
|
|
file.getSize(),
|
|
documents.get(0).getText(),
|
|
categoryId,
|
|
tags, chunkSize, overlap,
|
|
relativePath, folderId);
|
|
}
|
|
|
|
/**
|
|
* 保存上传的文件到本地磁盘
|
|
*
|
|
* @param file 上传的文件
|
|
* @return 相对路径
|
|
*/
|
|
private String saveFileToLocal(MultipartFile file) {
|
|
try {
|
|
String relativePath = fileStorageConfig.generateRelativePath(file.getOriginalFilename());
|
|
java.nio.file.Path targetPath = fileStorageConfig.getFilePath(relativePath);
|
|
|
|
// 确保父目录存在
|
|
java.io.File parentDir = targetPath.getParent().toFile();
|
|
if (!parentDir.exists()) {
|
|
parentDir.mkdirs();
|
|
}
|
|
|
|
// 使用 transferTo 保存文件（这是唯一一次调用）
|
|
file.transferTo(targetPath.toFile());
|
|
log.info("文件已保存到本地: {}", targetPath);
|
|
return relativePath;
|
|
} catch (java.io.IOException e) {
|
|
log.error("保存文件到本地失败", e);
|
|
throw new RuntimeException("保存文件失败: " + e.getMessage(), e);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* 解析字符串并上传（无原始文件）
|
|
*/
|
|
public KnowledgeDocument uploadString(String content, String title, Long categoryId, Long folderId,
|
|
List<String> tags, Integer chunkSize, Integer overlap) {
|
|
List<Document> documents = simpleStringDocumentReader.read(content);
|
|
return uploadDocument(documents, title, title, "txt",
|
|
(long) content.length(), content, categoryId, tags, chunkSize, overlap, null, folderId);
|
|
}
|
|
|
|
/**
|
|
* 解析 Markdown 文件并上传
|
|
*/
|
|
public KnowledgeDocument uploadMarkdown(MultipartFile file, String title, Long categoryId, Long folderId,
|
|
List<String> tags, Integer chunkSize, Integer overlap) {
|
|
// 1. 先保存原始文件
|
|
String relativePath = saveFileToLocal(file);
|
|
java.io.File savedFile = fileStorageConfig.getFilePath(relativePath).toFile();
|
|
|
|
// 2. 使用保存的文件进行解析
|
|
List<Document> documents = markdownDocumentLoader.loadMarkdownFromFile(savedFile);
|
|
String content = documents.stream().map(Document::getText).collect(Collectors.joining("\n"));
|
|
|
|
return uploadDocument(documents,
|
|
title != null ? title : file.getOriginalFilename(),
|
|
file.getOriginalFilename(),
|
|
"md",
|
|
file.getSize(),
|
|
content,
|
|
categoryId,
|
|
tags, chunkSize, overlap,
|
|
relativePath, folderId);
|
|
}
|
|
|
|
/**
|
|
* 解析 JSON 文件（基本方式）并上传
|
|
*/
|
|
public KnowledgeDocument uploadJsonBasic(MultipartFile file, String title, Long categoryId, Long folderId,
|
|
List<String> tags, Integer chunkSize, Integer overlap) {
|
|
// 1. 先保存原始文件
|
|
String relativePath = saveFileToLocal(file);
|
|
java.io.File savedFile = fileStorageConfig.getFilePath(relativePath).toFile();
|
|
|
|
// 2. 使用保存的文件进行解析
|
|
List<Document> documents = jsonDocumentLoader.loadBasicJsonFromFile(savedFile);
|
|
String content = documents.stream().map(Document::getText).collect(Collectors.joining("\n"));
|
|
|
|
return uploadDocument(documents,
|
|
title != null ? title : file.getOriginalFilename(),
|
|
file.getOriginalFilename(),
|
|
"json",
|
|
file.getSize(),
|
|
content,
|
|
categoryId,
|
|
tags, chunkSize, overlap,
|
|
relativePath, folderId);
|
|
}
|
|
|
|
/**
|
|
* 解析 JSON 文件（按字段）并上传
|
|
*/
|
|
public KnowledgeDocument uploadJsonFields(MultipartFile file, List<String> fields, String title,
|
|
Long categoryId, Long folderId, List<String> tags,
|
|
Integer chunkSize, Integer overlap) {
|
|
// 1. 先保存原始文件
|
|
String relativePath = saveFileToLocal(file);
|
|
java.io.File savedFile = fileStorageConfig.getFilePath(relativePath).toFile();
|
|
|
|
// 2. 使用保存的文件进行解析
|
|
List<Document> documents = jsonDocumentLoader.loadJsonByFieldsFromFile(savedFile, fields.toArray(new String[0]));
|
|
String content = documents.stream().map(Document::getText).collect(Collectors.joining("\n"));
|
|
|
|
return uploadDocument(documents,
|
|
title != null ? title : file.getOriginalFilename(),
|
|
file.getOriginalFilename(),
|
|
"json",
|
|
file.getSize(),
|
|
content,
|
|
categoryId,
|
|
tags, chunkSize, overlap,
|
|
relativePath, folderId);
|
|
}
|
|
|
|
/**
|
|
* 解析 JSON 文件（按指针）并上传
|
|
*/
|
|
public KnowledgeDocument uploadJsonPointer(MultipartFile file, String pointer, String title,
|
|
Long categoryId, Long folderId, List<String> tags,
|
|
Integer chunkSize, Integer overlap) {
|
|
// 1. 先保存原始文件
|
|
String relativePath = saveFileToLocal(file);
|
|
java.io.File savedFile = fileStorageConfig.getFilePath(relativePath).toFile();
|
|
|
|
// 2. 使用保存的文件进行解析
|
|
List<Document> documents = jsonDocumentLoader.loadJsonByPointerFromFile(savedFile, pointer);
|
|
String content = documents.stream().map(Document::getText).collect(Collectors.joining("\n"));
|
|
|
|
return uploadDocument(documents,
|
|
title != null ? title : file.getOriginalFilename(),
|
|
file.getOriginalFilename(),
|
|
"json",
|
|
file.getSize(),
|
|
content,
|
|
categoryId,
|
|
tags, chunkSize, overlap,
|
|
relativePath, folderId);
|
|
}
|
|
|
|
/**
|
|
* 文件夹批量上传：按相对路径解析子目录结构，逐文件上传
|
|
*
|
|
* @param files 文件列表
|
|
* @param relativePaths 每个文件对应的相对路径（如 "dir1/dir2/file.txt"）
|
|
* @param categoryId 分类ID（可选，baseFolderId 为 0 时作为目标分类）
|
|
* @param folderId 基础目录ID（可选，null/0 表示从分类根目录开始）
|
|
* @param tags 标签列表（可选）
|
|
* @param chunkSize 分块大小（可选）
|
|
* @param overlap 重叠大小（可选）
|
|
* @return 批量上传结果（successCount/failCount/details）
|
|
*/
|
|
public Map<String, Object> uploadFolder(List<MultipartFile> files, List<String> relativePaths,
|
|
Long categoryId, Long folderId, List<String> tags,
|
|
Integer chunkSize, Integer overlap) {
|
|
if (files == null || files.isEmpty() || relativePaths == null || relativePaths.isEmpty()) {
|
|
throw new RuntimeException("文件和路径列表不能为空");
|
|
}
|
|
if (files.size() != relativePaths.size()) {
|
|
throw new RuntimeException("文件与路径数量不一致");
|
|
}
|
|
if (files.size() > 500) {
|
|
throw new RuntimeException("单次文件夹上传最多支持 500 个文件");
|
|
}
|
|
|
|
// 解析目标分类：基础目录存在时以其所属分类为准，否则使用入参分类
|
|
Long baseFolderId = folderId != null ? folderId : 0L;
|
|
Long targetCategoryId;
|
|
if (baseFolderId > 0) {
|
|
KnowledgeFolder base = folderService.getFolderById(baseFolderId);
|
|
if (base == null) {
|
|
throw new RuntimeException("目录不存在");
|
|
}
|
|
targetCategoryId = base.getCategoryId();
|
|
} else {
|
|
targetCategoryId = categoryId;
|
|
}
|
|
|
|
int successCount = 0;
|
|
int failCount = 0;
|
|
List<Map<String, Object>> details = new ArrayList<>();
|
|
|
|
for (int i = 0; i < files.size(); i++) {
|
|
String originalPath = relativePaths.get(i);
|
|
try {
|
|
String sanitized = sanitizeRelativePath(originalPath);
|
|
List<String> segs = new ArrayList<>(List.of(sanitized.split("/")));
|
|
// 目录段 = 去掉最后一段（文件名）
|
|
List<String> dirSegs = segs.subList(0, segs.size() - 1);
|
|
|
|
Long targetFolderId;
|
|
if (!dirSegs.isEmpty()) {
|
|
if (targetCategoryId == null || targetCategoryId == 0) {
|
|
throw new RuntimeException("上传含子目录时必须先选择分类");
|
|
}
|
|
targetFolderId = folderService.ensureFolderPath(targetCategoryId, baseFolderId, dirSegs);
|
|
} else {
|
|
targetFolderId = baseFolderId;
|
|
}
|
|
uploadFile(files.get(i), null, targetCategoryId, targetFolderId, tags, chunkSize, overlap);
|
|
successCount++;
|
|
} catch (Exception e) {
|
|
failCount++;
|
|
details.add(Map.of("file", originalPath, "error", e.getMessage()));
|
|
log.warn("文件夹上传单个文件失败: file={}", originalPath, e);
|
|
}
|
|
}
|
|
|
|
Map<String, Object> result = new HashMap<>();
|
|
result.put("successCount", successCount);
|
|
result.put("failCount", failCount);
|
|
result.put("details", details);
|
|
return result;
|
|
}
|
|
|
|
/**
|
|
* 净化相对路径：反斜杠转正斜杠、去开头斜杠，校验非法段
|
|
*
|
|
* @param path 原始相对路径
|
|
* @return 净化后的路径
|
|
*/
|
|
private String sanitizeRelativePath(String path) {
|
|
if (path == null || path.isEmpty()) {
|
|
throw new RuntimeException("文件相对路径不能为空");
|
|
}
|
|
String normalized = path.replace('\\', '/');
|
|
while (normalized.startsWith("/")) {
|
|
normalized = normalized.substring(1);
|
|
}
|
|
if (normalized.isEmpty()) {
|
|
throw new RuntimeException("文件相对路径不能为空");
|
|
}
|
|
String[] parts = normalized.split("/");
|
|
for (String part : parts) {
|
|
if (part.isEmpty()) {
|
|
throw new RuntimeException("文件相对路径包含空目录段: " + path);
|
|
}
|
|
if (".".equals(part) || "..".equals(part)) {
|
|
throw new RuntimeException("文件相对路径包含非法目录段: " + path);
|
|
}
|
|
if (part.length() > 255) {
|
|
throw new RuntimeException("目录名超过 255 字符限制: " + part);
|
|
}
|
|
}
|
|
if (parts.length > 32) {
|
|
throw new RuntimeException("目录层级超过 32 层限制");
|
|
}
|
|
return normalized;
|
|
}
|
|
|
|
// ==================== 文件下载 ====================
|
|
|
|
/**
|
|
* 获取文档的原始文件（用于下载）
|
|
*
|
|
* @param documentId 文档ID
|
|
* @return 文件对象和原始文件名
|
|
*/
|
|
public java.util.Map.Entry<java.io.File, String> getDocumentFile(Long documentId) {
|
|
KnowledgeDocument doc = documentMapper.selectById(documentId);
|
|
if (doc == null || doc.isDelete()) {
|
|
throw new RuntimeException("文档不存在");
|
|
}
|
|
if (doc.getFilePath() == null || doc.getFilePath().isEmpty()) {
|
|
throw new RuntimeException("该文档无原始文件（纯文本上传）");
|
|
}
|
|
|
|
java.io.File file = fileStorageConfig.getFilePath(doc.getFilePath()).toFile();
|
|
if (!file.exists()) {
|
|
throw new RuntimeException("文件不存在或已被删除");
|
|
}
|
|
|
|
return java.util.Map.entry(file, doc.getSourceName());
|
|
}
|
|
|
|
// ==================== 文档管理 ====================
|
|
|
|
/**
|
|
* 分页查询文档列表（手动分页，支持关键词搜索 + 标签筛选 + 目录筛选）
|
|
*/
|
|
public Map<String, Object> listDocuments(int page, int size, Long categoryId, Long folderId, String status,
|
|
String keyword, String tag, String sortField, String sortOrder) {
|
|
// 参数安全校验
|
|
if (page < 1) page = 1;
|
|
if (size < 1 || size > 100) size = 10;
|
|
// LIKE 通配符转义
|
|
String kw = (keyword != null && !keyword.trim().isEmpty())
|
|
? keyword.trim().replace("%", "\\%").replace("_", "\\_") : null;
|
|
|
|
// 构建基础条件（用于 count 和 list）
|
|
QueryWrapper<KnowledgeDocument> countWrapper = new QueryWrapper<>();
|
|
if (categoryId != null && categoryId > 0) {
|
|
countWrapper.eq("category_id", categoryId);
|
|
}
|
|
if (folderId != null) {
|
|
countWrapper.eq("folder_id", folderId);
|
|
}
|
|
if (status != null && !status.isEmpty()) {
|
|
countWrapper.eq("status", status);
|
|
}
|
|
if (kw != null) {
|
|
countWrapper.and(w -> w.like("title", kw).or().like("source_name", kw));
|
|
}
|
|
// P1-2.3: 标签筛选（JSONB 包含查询，使用 ObjectMapper 构建合法 JSON）
|
|
if (tag != null && !tag.trim().isEmpty()) {
|
|
try {
|
|
ObjectMapper om = new ObjectMapper();
|
|
String jsonArray = om.writeValueAsString(List.of(tag.trim()));
|
|
countWrapper.apply("tags->'tags' @> {0}::jsonb", jsonArray);
|
|
} catch (Exception e) {
|
|
log.warn("构建标签筛选 JSON 失败: tag={}", tag, e);
|
|
}
|
|
}
|
|
|
|
// 先查询总数（不加 ORDER BY）
|
|
Long total = documentMapper.selectCount(countWrapper);
|
|
|
|
// 构建列表查询条件
|
|
QueryWrapper<KnowledgeDocument> listWrapper = new QueryWrapper<>();
|
|
if (categoryId != null && categoryId > 0) {
|
|
listWrapper.eq("category_id", categoryId);
|
|
}
|
|
if (folderId != null) {
|
|
listWrapper.eq("folder_id", folderId);
|
|
}
|
|
if (status != null && !status.isEmpty()) {
|
|
listWrapper.eq("status", status);
|
|
}
|
|
if (kw != null) {
|
|
listWrapper.and(w -> w.like("title", kw).or().like("source_name", kw));
|
|
}
|
|
if (tag != null && !tag.trim().isEmpty()) {
|
|
try {
|
|
ObjectMapper om = new ObjectMapper();
|
|
String jsonArray = om.writeValueAsString(List.of(tag.trim()));
|
|
listWrapper.apply("tags->'tags' @> {0}::jsonb", jsonArray);
|
|
} catch (Exception e) {
|
|
log.warn("构建标签筛选 JSON 失败: tag={}", tag, e);
|
|
}
|
|
}
|
|
// 排序字段白名单（前端 colKey -> 数据库列名），防 SQL 注入
|
|
Map<String, String> sortColumns = Map.of(
|
|
"title", "title",
|
|
"fileType", "file_type",
|
|
"fileSize", "file_size",
|
|
"chunkCount", "chunk_count",
|
|
"createTime", "create_time");
|
|
// sortField 可能为 null，需先判空，避免不可变 Map 的 getOrDefault(null) 触发 NPE
|
|
String sortColumn = sortField != null ? sortColumns.getOrDefault(sortField, "create_time") : "create_time";
|
|
if ("asc".equalsIgnoreCase(sortOrder)) {
|
|
listWrapper.orderByAsc(sortColumn);
|
|
} else {
|
|
listWrapper.orderByDesc(sortColumn);
|
|
}
|
|
listWrapper.last("LIMIT " + size + " OFFSET " + (page - 1) * size);
|
|
List<KnowledgeDocument> records = documentMapper.selectList(listWrapper);
|
|
|
|
Map<String, Object> result = new HashMap<>();
|
|
result.put("records", records);
|
|
result.put("total", total);
|
|
result.put("page", page);
|
|
result.put("size", size);
|
|
result.put("pages", (total + size - 1) / size);
|
|
return result;
|
|
}
|
|
|
|
/**
|
|
* 获取文档详情
|
|
*/
|
|
public KnowledgeDocument getDocumentDetail(Long id) {
|
|
return documentMapper.selectById(id);
|
|
}
|
|
|
|
/**
|
|
* 获取文档的所有分块
|
|
*/
|
|
public List<Map<String, Object>> getDocumentChunks(Long id) {
|
|
String documentId = String.valueOf(id);
|
|
String sql = "SELECT id::text as id, content, metadata, " +
|
|
"COALESCE((metadata->>'chunkIndex')::int, 0) as chunk_index, " +
|
|
"create_time FROM vector_store " +
|
|
"WHERE metadata->>'documentId' = ? " +
|
|
" OR metadata->>'document_id' = ? " +
|
|
" OR metadata->>'docId' = ? " +
|
|
"ORDER BY " + chunkOrderSql();
|
|
List<Map<String, Object>> chunks = jdbcTemplate.queryForList(sql, documentId, documentId, documentId);
|
|
if (!chunks.isEmpty()) {
|
|
return chunks;
|
|
}
|
|
|
|
KnowledgeDocument doc = documentMapper.selectById(id);
|
|
if (doc == null) {
|
|
return chunks;
|
|
}
|
|
return getDocumentChunksByTitleFallback(doc);
|
|
}
|
|
|
|
private List<Map<String, Object>> getDocumentChunksByTitleFallback(KnowledgeDocument doc) {
|
|
List<String> filters = new ArrayList<>();
|
|
List<Object> params = new ArrayList<>();
|
|
if (doc.getSourceName() != null && !doc.getSourceName().isBlank()) {
|
|
filters.add("metadata->>'sourceName' = ?");
|
|
params.add(doc.getSourceName());
|
|
}
|
|
if (doc.getTitle() != null && !doc.getTitle().isBlank()) {
|
|
filters.add("metadata->>'title' = ?");
|
|
params.add(doc.getTitle());
|
|
}
|
|
if (filters.isEmpty()) {
|
|
return List.of();
|
|
}
|
|
|
|
String sql = "SELECT id::text as id, content, metadata, " +
|
|
"COALESCE((metadata->>'chunkIndex')::int, 0) as chunk_index, " +
|
|
"create_time FROM vector_store " +
|
|
"WHERE " + String.join(" OR ", filters) + " " +
|
|
"ORDER BY " + chunkOrderSql();
|
|
if (doc.getChunkCount() != null && doc.getChunkCount() > 0) {
|
|
sql += " LIMIT ?";
|
|
params.add(doc.getChunkCount());
|
|
}
|
|
return jdbcTemplate.queryForList(sql, params.toArray());
|
|
}
|
|
|
|
private String chunkOrderSql() {
|
|
return "CASE WHEN metadata->>'chunkIndex' ~ '^[0-9]+$' " +
|
|
"THEN (metadata->>'chunkIndex')::int ELSE 0 END ASC, create_time ASC, id::text ASC";
|
|
}
|
|
|
|
/**
|
|
* 删除文档（逻辑删除 + 级联删除向量）
|
|
*/
|
|
@Transactional(rollbackFor = Exception.class)
|
|
public int deleteDocument(Long id) {
|
|
KnowledgeDocument doc = documentMapper.selectById(id);
|
|
if (doc == null) {
|
|
throw new RuntimeException("文档不存在");
|
|
}
|
|
// 删除关联的向量
|
|
int vectorCount = deleteVectorsByDocumentId(String.valueOf(id));
|
|
// 逻辑删除文档记录
|
|
documentMapper.deleteById(id);
|
|
log.info("删除文档: id={}, title={}, 删除向量数={}", id, doc.getTitle(), vectorCount);
|
|
return vectorCount;
|
|
}
|
|
|
|
/**
|
|
* 批量删除文档
|
|
*
|
|
* @param ids 文档ID列表
|
|
* @return 批量操作结果
|
|
*/
|
|
public Map<String, Object> batchDeleteDocuments(List<Long> ids) {
|
|
int successCount = 0;
|
|
int failCount = 0;
|
|
List<Map<String, Object>> details = new ArrayList<>();
|
|
|
|
for (Long id : ids) {
|
|
try {
|
|
int vectorCount = deleteDocument(id);
|
|
successCount++;
|
|
details.add(Map.of("id", id.toString(), "success", true, "deletedVectors", vectorCount));
|
|
} catch (Exception e) {
|
|
failCount++;
|
|
details.add(Map.of("id", id.toString(), "success", false, "message", e.getMessage()));
|
|
log.warn("批量删除文档失败: id={}", id, e);
|
|
}
|
|
}
|
|
|
|
Map<String, Object> result = new HashMap<>();
|
|
result.put("successCount", successCount);
|
|
result.put("failCount", failCount);
|
|
result.put("details", details);
|
|
return result;
|
|
}
|
|
|
|
/**
|
|
* 批量重新处理文档
|
|
*
|
|
* @param ids 文档ID列表
|
|
* @return 批量操作结果
|
|
*/
|
|
public Map<String, Object> batchReprocessDocuments(List<Long> ids) {
|
|
int successCount = 0;
|
|
int failCount = 0;
|
|
List<Map<String, Object>> details = new ArrayList<>();
|
|
|
|
for (Long id : ids) {
|
|
try {
|
|
reprocessDocument(id);
|
|
successCount++;
|
|
details.add(Map.of("id", id.toString(), "success", true));
|
|
} catch (Exception e) {
|
|
failCount++;
|
|
details.add(Map.of("id", id.toString(), "success", false, "message", e.getMessage()));
|
|
log.warn("批量重新处理文档失败: id={}", id, e);
|
|
}
|
|
}
|
|
|
|
Map<String, Object> result = new HashMap<>();
|
|
result.put("successCount", successCount);
|
|
result.put("failCount", failCount);
|
|
result.put("details", details);
|
|
return result;
|
|
}
|
|
|
|
/**
|
|
* 重新处理文档（异步：先标记为 PROCESSING，后台重新分块 + 向量化）
|
|
*/
|
|
@Transactional(rollbackFor = Exception.class)
|
|
public KnowledgeDocument reprocessDocument(Long id) {
|
|
KnowledgeDocument doc = documentMapper.selectById(id);
|
|
if (doc == null) {
|
|
throw new RuntimeException("文档不存在");
|
|
}
|
|
if (doc.getContent() == null || doc.getContent().isEmpty()) {
|
|
throw new RuntimeException("文档无内容，无法重新处理");
|
|
}
|
|
|
|
// 标记为 PROCESSING，清空旧状态
|
|
doc.setStatus("PROCESSING");
|
|
doc.setChunkCount(0);
|
|
doc.setErrorMessage(null);
|
|
documentMapper.updateById(doc);
|
|
|
|
// 解析内容并触发异步处理
|
|
List<Document> documents = simpleStringDocumentReader.read(doc.getContent());
|
|
documentProcessingService.reprocessDocumentAsync(id, documents);
|
|
|
|
log.info("文档重新处理已提交: id={}, title={}", doc.getId(), doc.getTitle());
|
|
return doc;
|
|
}
|
|
|
|
/**
|
|
* 更新文档元信息
|
|
*/
|
|
public void updateDocumentMetadata(Long id, String title, Long categoryId, List<String> tags) {
|
|
KnowledgeDocument doc = documentMapper.selectById(id);
|
|
if (doc == null) {
|
|
throw new RuntimeException("文档不存在");
|
|
}
|
|
if (title != null && !title.isEmpty()) {
|
|
doc.setTitle(title);
|
|
}
|
|
if (categoryId != null) {
|
|
doc.setCategoryId(categoryId);
|
|
}
|
|
if (tags != null) {
|
|
doc.setTags(Map.of("tags", tags));
|
|
}
|
|
documentMapper.updateById(doc);
|
|
|
|
// 同步更新 vector_store 中对应的 metadata
|
|
// 注意：Spring AI 当前没有直接更新 metadata 的 API
|
|
// 这里我们先更新文档记录，metadata 的同步留到后续优化
|
|
log.info("更新文档元信息: id={}, title={}", id, doc.getTitle());
|
|
}
|
|
|
|
// ==================== 启用/禁用 ====================
|
|
|
|
/**
|
|
* P1-2.1: 切换文档启用/禁用状态
|
|
* 禁用后不参与 RAG 检索但保留数据，同步更新 vector_store metadata
|
|
* 使用原子翻转 SQL 避免并发竞态
|
|
*/
|
|
public KnowledgeDocument toggleDocument(Long id) {
|
|
// 原子翻转：直接在 DB 层 NOT enabled，避免先 SELECT 再 UPDATE 的竞态问题
|
|
int updated = jdbcTemplate.update(
|
|
"UPDATE knowledge_document SET enabled = NOT enabled WHERE id = ? AND is_delete = false",
|
|
id);
|
|
if (updated == 0) {
|
|
throw new RuntimeException("文档不存在或已被删除");
|
|
}
|
|
|
|
// 查询最新状态
|
|
KnowledgeDocument doc = documentMapper.selectById(id);
|
|
if (doc == null) {
|
|
throw new RuntimeException("文档不存在");
|
|
}
|
|
|
|
// 同步更新 vector_store 中的 enabled metadata
|
|
syncVectorEnabledMetadata(String.valueOf(id), Boolean.TRUE.equals(doc.getEnabled()));
|
|
log.info("文档状态切换: id={}, enabled={}", id, doc.getEnabled());
|
|
return doc;
|
|
}
|
|
|
|
/**
|
|
* P1-2.1: 批量启用/禁用文档
|
|
*/
|
|
public Map<String, Object> batchToggleDocuments(List<Long> ids, boolean enabled) {
|
|
int successCount = 0;
|
|
int failCount = 0;
|
|
for (Long id : ids) {
|
|
try {
|
|
KnowledgeDocument doc = documentMapper.selectById(id);
|
|
if (doc == null) {
|
|
failCount++;
|
|
continue;
|
|
}
|
|
doc.setEnabled(enabled);
|
|
documentMapper.updateById(doc);
|
|
syncVectorEnabledMetadata(String.valueOf(id), enabled);
|
|
successCount++;
|
|
} catch (Exception e) {
|
|
failCount++;
|
|
log.warn("批量切换文档状态失败: id={}", id, e);
|
|
}
|
|
}
|
|
Map<String, Object> result = new HashMap<>();
|
|
result.put("successCount", successCount);
|
|
result.put("failCount", failCount);
|
|
return result;
|
|
}
|
|
|
|
/**
|
|
* 同步更新 vector_store 中指定文档所有分块的 enabled metadata
|
|
* 使用 JdbcTemplate 直接 UPDATE JSON 字段
|
|
* 注意：Spring AI 的 PgVectorStore 自动建表时 metadata 是 json 类型，不是 jsonb
|
|
* 统一写入字符串类型 "true"/"false"，与 searchDocuments 的过滤条件保持一致
|
|
*/
|
|
private void syncVectorEnabledMetadata(String documentId, boolean enabled) {
|
|
// 使用 to_jsonb(?::text) 将字符串转换为 JSONB 字符串值，与 searchDocuments 过滤条件一致
|
|
String sql = "UPDATE vector_store SET metadata = (jsonb_set(metadata::jsonb, '{enabled}', " +
|
|
"to_jsonb(?::text)))::json WHERE metadata->>'documentId' = ?";
|
|
jdbcTemplate.update(sql, enabled ? "true" : "false", documentId);
|
|
}
|
|
|
|
// ==================== 移动文档（分类根 / 目录） ====================
|
|
|
|
/**
|
|
* 统一移动文档：目标既可以是「分类根」（folderId=0），也可以是「具体目录」（folderId>0）。
|
|
* <p>约定与上传一致：目标为目录时，强制使用目录所属分类覆盖 categoryId；
|
|
* 目标为分类根时，使用传入的 categoryId（null/0 表示未分类）。</p>
|
|
* <p>categoryId 变更时同步 vector_store 的 categoryId metadata，保证 RAG 分类过滤正确。</p>
|
|
*/
|
|
@Transactional(rollbackFor = Exception.class)
|
|
public int moveDocuments(List<Long> ids, Long categoryId, Long folderId) {
|
|
Long targetFolderId = (folderId != null && folderId > 0) ? folderId : 0L;
|
|
Long targetCategoryId;
|
|
if (targetFolderId > 0) {
|
|
KnowledgeFolder folder = folderService.getFolderById(targetFolderId);
|
|
if (folder == null) {
|
|
throw new RuntimeException("目标目录不存在");
|
|
}
|
|
targetCategoryId = folder.getCategoryId();
|
|
} else {
|
|
targetCategoryId = (categoryId != null) ? categoryId : 0L;
|
|
}
|
|
|
|
int updated = 0;
|
|
for (Long id : ids) {
|
|
KnowledgeDocument doc = documentMapper.selectById(id);
|
|
if (doc == null) {
|
|
continue;
|
|
}
|
|
Long oldCategoryId = doc.getCategoryId();
|
|
doc.setFolderId(targetFolderId);
|
|
doc.setCategoryId(targetCategoryId);
|
|
documentMapper.updateById(doc);
|
|
// 分类变更时同步向量 metadata 的 categoryId（folderId 不参与 RAG，无需同步）
|
|
if (oldCategoryId == null || !oldCategoryId.equals(targetCategoryId)) {
|
|
syncVectorCategoryMetadata(String.valueOf(id), targetCategoryId);
|
|
}
|
|
updated++;
|
|
}
|
|
log.info("移动文档: ids={}, folderId={}, categoryId={}, 实际更新={}", ids, targetFolderId, targetCategoryId, updated);
|
|
return updated;
|
|
}
|
|
|
|
/**
|
|
* 同步 vector_store 中指定文档所有分块的 categoryId metadata。
|
|
* <p>与 {@link #syncVectorEnabledMetadata} 同构：metadata 列是 json 类型，用 jsonb 运算后转回 json。
|
|
* categoryId 在 DocumentProcessingService 中以字符串写入，故同步时写字符串；
|
|
* 目标为未分类（categoryId<=0）时移除该键（原写入逻辑仅在 categoryId>0 时写键）。</p>
|
|
*/
|
|
private void syncVectorCategoryMetadata(String documentId, Long categoryId) {
|
|
if (categoryId == null || categoryId <= 0) {
|
|
jdbcTemplate.update(
|
|
"UPDATE vector_store SET metadata = (metadata::jsonb - 'categoryId')::json WHERE metadata->>'documentId' = ?",
|
|
documentId);
|
|
} else {
|
|
jdbcTemplate.update(
|
|
"UPDATE vector_store SET metadata = (jsonb_set(metadata::jsonb, '{categoryId}', " +
|
|
"to_jsonb(?::text)))::json WHERE metadata->>'documentId' = ?",
|
|
String.valueOf(categoryId), documentId);
|
|
}
|
|
}
|
|
|
|
// ==================== 标签管理 ====================
|
|
|
|
/**
|
|
* P1-2.3: 获取标签列表（从所有文档的 tags JSONB 聚合去重，含使用次数）
|
|
*
|
|
* @return [{tag: "标签名", count: 使用次数}]
|
|
*/
|
|
public List<Map<String, Object>> getTagList() {
|
|
String sql = "SELECT tag_value AS tag, COUNT(*) AS count " +
|
|
"FROM knowledge_document, jsonb_array_elements(tags->'tags') AS tag_value " +
|
|
"WHERE is_delete = false AND tags IS NOT NULL AND tags->'tags' IS NOT NULL " +
|
|
"GROUP BY tag_value ORDER BY count DESC, tag_value ASC";
|
|
try {
|
|
return jdbcTemplate.queryForList(sql);
|
|
} catch (Exception e) {
|
|
log.warn("获取标签列表失败", e);
|
|
return List.of();
|
|
}
|
|
}
|
|
|
|
// ==================== 分块管理 ====================
|
|
|
|
/**
|
|
* P1-2.5: 更新单个分块内容（同步更新 vector_store 并重新向量化）
|
|
*/
|
|
@Transactional(rollbackFor = Exception.class)
|
|
public void updateChunk(Long docId, int chunkIndex, String newContent) {
|
|
KnowledgeDocument doc = documentMapper.selectById(docId);
|
|
if (doc == null) {
|
|
throw new RuntimeException("文档不存在");
|
|
}
|
|
|
|
String docIdStr = String.valueOf(docId);
|
|
// 查找对应的向量记录 ID
|
|
String selectSql = "SELECT id::text FROM vector_store " +
|
|
"WHERE (metadata->>'documentId' = ? OR metadata->>'document_id' = ?) " +
|
|
"AND metadata->>'chunkIndex' = ? " +
|
|
"LIMIT 1";
|
|
List<String> vectorIds = jdbcTemplate.queryForList(selectSql, String.class,
|
|
docIdStr, docIdStr, String.valueOf(chunkIndex));
|
|
|
|
if (vectorIds.isEmpty()) {
|
|
throw new RuntimeException("未找到对应的分块向量记录");
|
|
}
|
|
|
|
String vectorId = vectorIds.get(0);
|
|
|
|
// 删除旧向量
|
|
pgVectorVectorStore.delete(List.of(vectorId));
|
|
|
|
// 构建新文档并重新向量化
|
|
Map<String, Object> meta = new HashMap<>();
|
|
meta.put("documentId", docIdStr);
|
|
meta.put("chunkIndex", chunkIndex);
|
|
meta.put("sourceName", doc.getSourceName());
|
|
meta.put("title", doc.getTitle());
|
|
if (doc.getCategoryId() != null && doc.getCategoryId() > 0) {
|
|
meta.put("categoryId", String.valueOf(doc.getCategoryId()));
|
|
}
|
|
if (doc.getTags() != null && doc.getTags().containsKey("tags")) {
|
|
meta.put("tags", doc.getTags().get("tags"));
|
|
}
|
|
// 标记启用状态，与文档当前状态保持一致
|
|
meta.put("enabled", String.valueOf(Boolean.TRUE.equals(doc.getEnabled())));
|
|
|
|
Document newDoc = new Document(vectorId, newContent, meta);
|
|
pgVectorVectorStore.add(List.of(newDoc));
|
|
|
|
log.info("更新分块: docId={}, chunkIndex={}, vectorId={}", docId, chunkIndex, vectorId);
|
|
}
|
|
|
|
/**
|
|
* P1-2.5: 删除单个分块（同步从 vector_store 移除向量）
|
|
*/
|
|
@Transactional(rollbackFor = Exception.class)
|
|
public void deleteChunk(Long docId, int chunkIndex) {
|
|
KnowledgeDocument doc = documentMapper.selectById(docId);
|
|
if (doc == null) {
|
|
throw new RuntimeException("文档不存在");
|
|
}
|
|
|
|
String docIdStr = String.valueOf(docId);
|
|
// 查找对应的向量记录 ID
|
|
String selectSql = "SELECT id::text FROM vector_store " +
|
|
"WHERE (metadata->>'documentId' = ? OR metadata->>'document_id' = ?) " +
|
|
"AND metadata->>'chunkIndex' = ?";
|
|
List<String> vectorIds = jdbcTemplate.queryForList(selectSql, String.class,
|
|
docIdStr, docIdStr, String.valueOf(chunkIndex));
|
|
|
|
if (!vectorIds.isEmpty()) {
|
|
pgVectorVectorStore.delete(vectorIds);
|
|
}
|
|
|
|
// 更新文档的分块计数
|
|
doc.setChunkCount(Math.max(0, (doc.getChunkCount() != null ? doc.getChunkCount() : 1) - vectorIds.size()));
|
|
documentMapper.updateById(doc);
|
|
|
|
log.info("删除分块: docId={}, chunkIndex={}, 删除向量数={}", docId, chunkIndex, vectorIds.size());
|
|
}
|
|
|
|
// ==================== 语义搜索 ====================
|
|
|
|
/**
|
|
* 语义搜索
|
|
*/
|
|
public List<SearchResult> searchDocuments(String query, int topK, double similarityThreshold, Long categoryId) {
|
|
List<Long> categoryIds = categoryId != null ? List.of(categoryId) : Collections.emptyList();
|
|
return searchDocuments(query, topK, similarityThreshold, categoryIds);
|
|
}
|
|
|
|
/**
|
|
* 语义搜索（支持多个知识库分类过滤）
|
|
*/
|
|
public List<SearchResult> searchDocuments(String query, int topK, double similarityThreshold, List<Long> categoryIds) {
|
|
SearchRequest.Builder searchBuilder = SearchRequest.builder()
|
|
.query(query)
|
|
.topK(topK)
|
|
.similarityThreshold(similarityThreshold);
|
|
|
|
FilterExpressionBuilder builder = new FilterExpressionBuilder();
|
|
|
|
// P1-2.1: 始终过滤禁用的文档，只检索启用的
|
|
var enabledFilter = builder.eq("enabled", "true");
|
|
|
|
List<String> categoryIdStrings = normalizeCategoryIds(categoryIds);
|
|
if (!categoryIdStrings.isEmpty()) {
|
|
List<Object> values = categoryIdStrings.stream()
|
|
.map(value -> (Object) value)
|
|
.collect(Collectors.toList());
|
|
var categoryFilter = builder.in("categoryId", values);
|
|
searchBuilder.filterExpression(builder.and(enabledFilter, categoryFilter).build());
|
|
} else {
|
|
searchBuilder.filterExpression(enabledFilter.build());
|
|
}
|
|
|
|
List<Document> results;
|
|
try {
|
|
results = pgVectorVectorStore.similaritySearch(searchBuilder.build());
|
|
} catch (Exception e) {
|
|
if (categoryIdStrings.isEmpty()) {
|
|
throw e;
|
|
}
|
|
log.warn("向量检索分类过滤失败，改用本地 metadata 过滤兜底: categoryIds={}", categoryIdStrings, e);
|
|
results = pgVectorVectorStore.similaritySearch(SearchRequest.builder()
|
|
.query(query)
|
|
.topK(Math.max(topK * 5, 20))
|
|
.similarityThreshold(similarityThreshold)
|
|
.build())
|
|
.stream()
|
|
.filter(doc -> categoryIdStrings.contains(getStringFromMetadata(doc.getMetadata(), "categoryId")))
|
|
.filter(doc -> "true".equals(getStringFromMetadata(doc.getMetadata(), "enabled")))
|
|
.limit(topK)
|
|
.collect(Collectors.toList());
|
|
}
|
|
|
|
List<SearchResult> searchResults = new ArrayList<>();
|
|
for (Document doc : results) {
|
|
Map<String, Object> metadata = doc.getMetadata();
|
|
SearchResult result = SearchResult.builder()
|
|
.id(doc.getId())
|
|
.content(doc.getText())
|
|
.score(metadata.containsKey("distance") ? ((Number) metadata.get("distance")).doubleValue() : null)
|
|
.sourceName(getStringFromMetadata(metadata, "sourceName"))
|
|
.title(getStringFromMetadata(metadata, "title"))
|
|
.chunkIndex(getIntegerFromMetadata(metadata, "chunkIndex"))
|
|
.documentId(getStringFromMetadata(metadata, "documentId"))
|
|
.metadata(metadata)
|
|
.build();
|
|
searchResults.add(result);
|
|
}
|
|
|
|
return searchResults;
|
|
}
|
|
|
|
/**
|
|
* 多模式搜索（P0-001: 支持向量/关键词/混合检索）
|
|
*
|
|
* @param searchMode 检索模式：VECTOR（默认）/ KEYWORD / HYBRID
|
|
*/
|
|
public List<SearchResult> searchDocuments(String query, int topK, double similarityThreshold,
|
|
List<Long> categoryIds, String searchMode) {
|
|
if (searchMode == null || searchMode.isEmpty() || "VECTOR".equalsIgnoreCase(searchMode)) {
|
|
// 默认向量检索，向后兼容
|
|
List<SearchResult> results = searchDocuments(query, topK, similarityThreshold, categoryIds);
|
|
results.forEach(r -> r.setSearchMode("VECTOR"));
|
|
return results;
|
|
}
|
|
|
|
// 委托给 HybridSearchService 处理 KEYWORD 和 HYBRID 模式
|
|
if (hybridSearchService != null) {
|
|
com.wok.supportbot.rag.SearchMode mode;
|
|
try {
|
|
mode = com.wok.supportbot.rag.SearchMode.valueOf(searchMode.toUpperCase());
|
|
} catch (IllegalArgumentException e) {
|
|
mode = com.wok.supportbot.rag.SearchMode.VECTOR;
|
|
}
|
|
return hybridSearchService.search(query, mode, topK, similarityThreshold, categoryIds);
|
|
}
|
|
|
|
// HybridSearchService 不可用时降级到向量检索
|
|
log.warn("HybridSearchService 不可用，降级为向量检索");
|
|
return searchDocuments(query, topK, similarityThreshold, categoryIds);
|
|
}
|
|
|
|
@Autowired(required = false)
|
|
private com.wok.supportbot.rag.HybridSearchService hybridSearchService;
|
|
|
|
private List<String> normalizeCategoryIds(List<Long> categoryIds) {
|
|
return categoryFilter.normalize(categoryIds);
|
|
}
|
|
|
|
// ==================== 统计 ====================
|
|
|
|
/**
|
|
* 获取知识库统计信息
|
|
*/
|
|
public Map<String, Object> getStats() {
|
|
// 文档统计
|
|
Long totalDocuments = documentMapper.selectCount(null);
|
|
|
|
// 按文件类型统计
|
|
String typeSql = "SELECT file_type, COUNT(*) as count FROM knowledge_document WHERE is_delete = false GROUP BY file_type";
|
|
List<Map<String, Object>> typeStats = jdbcTemplate.queryForList(typeSql);
|
|
Map<String, Long> byFileType = typeStats.stream()
|
|
.collect(Collectors.toMap(
|
|
r -> (String) r.get("file_type"),
|
|
r -> ((Number) r.get("count")).longValue()
|
|
));
|
|
|
|
// 按分类统计
|
|
String catSql = "SELECT c.name, COUNT(d.id) as count FROM knowledge_document d " +
|
|
"LEFT JOIN knowledge_category c ON d.category_id = c.id " +
|
|
"WHERE d.is_delete = false GROUP BY c.name";
|
|
List<Map<String, Object>> catStats;
|
|
try {
|
|
catStats = jdbcTemplate.queryForList(catSql);
|
|
} catch (Exception e) {
|
|
catStats = new ArrayList<>();
|
|
}
|
|
|
|
// 向量总数
|
|
String vectorSql = "SELECT COUNT(*) FROM vector_store";
|
|
Long totalVectors;
|
|
try {
|
|
totalVectors = jdbcTemplate.queryForObject(vectorSql, Long.class);
|
|
} catch (Exception e) {
|
|
totalVectors = 0L;
|
|
}
|
|
|
|
// 最近上传时间
|
|
String lastUploadSql = "SELECT MAX(create_time) FROM knowledge_document WHERE is_delete = false";
|
|
Date lastUploadTime = jdbcTemplate.queryForObject(lastUploadSql, Date.class);
|
|
|
|
Map<String, Object> stats = new LinkedHashMap<>();
|
|
stats.put("totalDocuments", totalDocuments);
|
|
stats.put("totalVectors", totalVectors);
|
|
stats.put("lastUploadTime", lastUploadTime);
|
|
stats.put("byFileType", byFileType);
|
|
stats.put("byCategory", catStats);
|
|
|
|
return stats;
|
|
}
|
|
|
|
// ==================== 分类管理 ====================
|
|
|
|
/**
|
|
* 获取分类树
|
|
*/
|
|
public List<CategoryNode> getCategoryTree() {
|
|
List<KnowledgeCategory> categories = categoryMapper.selectList(
|
|
new QueryWrapper<KnowledgeCategory>().orderByAsc("sort_order"));
|
|
|
|
Map<Long, CategoryNode> nodeMap = new LinkedHashMap<>();
|
|
List<CategoryNode> rootNodes = new ArrayList<>();
|
|
|
|
for (KnowledgeCategory cat : categories) {
|
|
CategoryNode node = CategoryNode.builder()
|
|
.id(cat.getId())
|
|
.name(cat.getName())
|
|
.description(cat.getDescription())
|
|
.parentId(cat.getParentId())
|
|
.sortOrder(cat.getSortOrder())
|
|
.documentCount(cat.getDocumentCount())
|
|
.children(new ArrayList<>())
|
|
.build();
|
|
nodeMap.put(cat.getId(), node);
|
|
}
|
|
|
|
for (CategoryNode node : nodeMap.values()) {
|
|
if (node.getParentId() == null || node.getParentId() == 0) {
|
|
rootNodes.add(node);
|
|
} else {
|
|
CategoryNode parent = nodeMap.get(node.getParentId());
|
|
if (parent != null) {
|
|
parent.getChildren().add(node);
|
|
} else {
|
|
rootNodes.add(node);
|
|
}
|
|
}
|
|
}
|
|
|
|
return rootNodes;
|
|
}
|
|
|
|
/**
|
|
* 获取分类列表
|
|
*/
|
|
public List<KnowledgeCategory> listCategories() {
|
|
return categoryMapper.selectList(
|
|
new QueryWrapper<KnowledgeCategory>().orderByAsc("sort_order"));
|
|
}
|
|
|
|
/**
|
|
* 创建知识库分类
|
|
*/
|
|
public KnowledgeCategory createCategory(String name, String description, Long parentId, Integer sortOrder) {
|
|
KnowledgeCategory category = KnowledgeCategory.builder()
|
|
.name(name)
|
|
.description(description)
|
|
.parentId(parentId != null ? parentId : 0L)
|
|
.sortOrder(sortOrder != null ? sortOrder : 0)
|
|
.documentCount(0)
|
|
.build();
|
|
categoryMapper.insert(category);
|
|
return category;
|
|
}
|
|
|
|
/**
|
|
* 更新分类
|
|
*/
|
|
public void updateCategory(Long id, String name, String description, Integer sortOrder) {
|
|
KnowledgeCategory category = categoryMapper.selectById(id);
|
|
if (category == null) {
|
|
throw new RuntimeException("分类不存在");
|
|
}
|
|
if (name != null && !name.isEmpty()) {
|
|
category.setName(name);
|
|
}
|
|
if (description != null) {
|
|
category.setDescription(description);
|
|
}
|
|
if (sortOrder != null) {
|
|
category.setSortOrder(sortOrder);
|
|
}
|
|
categoryMapper.updateById(category);
|
|
}
|
|
|
|
/**
|
|
* 删除分类（不删除文档，仅清空关联；级联清理该分类下的目录）
|
|
*/
|
|
@Transactional(rollbackFor = Exception.class)
|
|
public void deleteCategory(Long id) {
|
|
// 级联清理该分类下的目录及其子孙目录（文档 folder_id 置 0，目录逻辑删除）
|
|
folderService.deleteFoldersByCategoryId(id);
|
|
|
|
// 将关联的文档 category_id 设为 0
|
|
KnowledgeDocument updateDoc = new KnowledgeDocument();
|
|
updateDoc.setCategoryId(0L);
|
|
documentMapper.update(updateDoc, new QueryWrapper<KnowledgeDocument>().eq("category_id", id));
|
|
|
|
// 逻辑删除分类
|
|
categoryMapper.deleteById(id);
|
|
}
|
|
|
|
// ==================== 内部方法 ====================
|
|
|
|
/**
|
|
* 计算文本内容的 SHA-256 哈希值
|
|
*/
|
|
private String computeContentHash(String content) {
|
|
if (content == null || content.isEmpty()) {
|
|
return null;
|
|
}
|
|
try {
|
|
MessageDigest digest = MessageDigest.getInstance("SHA-256");
|
|
byte[] hashBytes = digest.digest(content.getBytes(StandardCharsets.UTF_8));
|
|
StringBuilder sb = new StringBuilder();
|
|
for (byte b : hashBytes) {
|
|
sb.append(String.format("%02x", b));
|
|
}
|
|
return sb.toString();
|
|
} catch (NoSuchAlgorithmException e) {
|
|
log.error("SHA-256 算法不可用", e);
|
|
return null;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* 检查内容是否重复
|
|
* @param contentHash 内容哈希值
|
|
* @param categoryId 分类ID
|
|
* @param folderId 目录ID（null/0 表示分类根目录）
|
|
* @return 重复文档的标题，如果不存在重复则返回 null
|
|
*/
|
|
private String checkContentDuplicate(String contentHash, Long categoryId, Long folderId) {
|
|
if (contentHash == null) {
|
|
return null;
|
|
}
|
|
QueryWrapper<KnowledgeDocument> wrapper = new QueryWrapper<>();
|
|
wrapper.eq("content_hash", contentHash);
|
|
wrapper.eq("category_id", categoryId != null ? categoryId : 0L);
|
|
wrapper.eq("folder_id", folderId != null ? folderId : 0L);
|
|
wrapper.select("title");
|
|
List<KnowledgeDocument> existing = documentMapper.selectList(wrapper);
|
|
return existing != null && !existing.isEmpty() ? existing.get(0).getTitle() : null;
|
|
}
|
|
|
|
/**
|
|
* 根据文档ID删除 vector_store 中关联的所有向量
|
|
*/
|
|
private int deleteVectorsByDocumentId(String documentId) {
|
|
String sql = "SELECT id::text FROM vector_store WHERE metadata->>'documentId' = ?";
|
|
List<String> ids = jdbcTemplate.queryForList(sql, String.class, documentId);
|
|
|
|
if (!ids.isEmpty()) {
|
|
pgVectorVectorStore.delete(ids);
|
|
log.debug("删除向量: documentId={}, count={}", documentId, ids.size());
|
|
}
|
|
return ids.size();
|
|
}
|
|
|
|
/**
|
|
* 获取文件扩展名
|
|
*/
|
|
/**
|
|
* 取文件名最后一段，剥离可能的路径前缀。
|
|
* 文件夹上传（webkitdirectory）时浏览器 filename 携带相对路径（含根目录名），
|
|
* 直接用作标题/源文件名会导致带「目录/」前缀。
|
|
*/
|
|
private String stripPath(String filename) {
|
|
if (filename == null || filename.isEmpty()) {
|
|
return filename;
|
|
}
|
|
String normalized = filename.replace('\\', '/');
|
|
int idx = normalized.lastIndexOf('/');
|
|
return idx >= 0 ? normalized.substring(idx + 1) : normalized;
|
|
}
|
|
|
|
private String getFileExtension(String filename) {
|
|
if (filename == null || !filename.contains(".")) {
|
|
return "unknown";
|
|
}
|
|
return filename.substring(filename.lastIndexOf(".") + 1).toLowerCase();
|
|
}
|
|
|
|
/**
|
|
* 从 metadata 中安全获取字符串值
|
|
*/
|
|
private String getStringFromMetadata(Map<String, Object> metadata, String key) {
|
|
Object value = metadata.get(key);
|
|
return value != null ? value.toString() : null;
|
|
}
|
|
|
|
/**
|
|
* 从 metadata 中安全获取整数值
|
|
*/
|
|
private Integer getIntegerFromMetadata(Map<String, Object> metadata, String key) {
|
|
Object value = metadata.get(key);
|
|
if (value == null) return null;
|
|
if (value instanceof Number) {
|
|
return ((Number) value).intValue();
|
|
}
|
|
try {
|
|
return Integer.parseInt(value.toString());
|
|
} catch (NumberFormatException e) {
|
|
return null;
|
|
}
|
|
}
|
|
}
|