package com.wok.supportbot.service; import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; import com.wok.supportbot.dao.KnowledgeCategoryMapper; import com.wok.supportbot.dao.KnowledgeDocumentMapper; import com.wok.supportbot.document.extract.JsonDocumentLoader; import com.wok.supportbot.document.extract.MarkdownDocumentLoader; import com.wok.supportbot.document.extract.SimpleStringDocumentReader; import com.wok.supportbot.document.extract.TikaDocumentReader; import com.wok.supportbot.document.transform.MyKeywordEnricher; import com.wok.supportbot.document.transform.MyTokenTextSplitter; import com.wok.supportbot.entity.CategoryNode; import com.wok.supportbot.entity.KnowledgeCategory; import com.wok.supportbot.entity.KnowledgeDocument; import com.wok.supportbot.entity.SearchResult; import lombok.extern.slf4j.Slf4j; import org.springframework.ai.document.Document; import org.springframework.ai.vectorstore.SearchRequest; import org.springframework.ai.vectorstore.VectorStore; import org.springframework.ai.vectorstore.filter.FilterExpressionBuilder; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; import org.springframework.web.multipart.MultipartFile; import com.fasterxml.jackson.databind.ObjectMapper; import java.nio.charset.StandardCharsets; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.util.*; import java.util.stream.Collectors; /** * 知识库文档管理服务 * 统一管理文档的上传、删除、搜索、统计、分类等操作 */ @Service @Slf4j public class DocumentService { @Autowired private KnowledgeDocumentMapper documentMapper; @Autowired private KnowledgeCategoryMapper categoryMapper; @Autowired private JdbcTemplate jdbcTemplate; @Autowired private VectorStore pgVectorVectorStore; @Autowired private MyTokenTextSplitter myTokenTextSplitter; @Autowired private MyKeywordEnricher myKeywordEnricher; @Autowired private TikaDocumentReader tikaDocumentReader; @Autowired private SimpleStringDocumentReader simpleStringDocumentReader; @Autowired private MarkdownDocumentLoader markdownDocumentLoader; @Autowired private JsonDocumentLoader jsonDocumentLoader; @Autowired private DocumentProcessingService documentProcessingService; // ==================== 文档上传 ==================== /** * 统一文档上传流程:创建记录(PROCESSING)→ 触发异步处理 * 同步阶段:内容去重检查 + 创建 PROCESSING 记录 * 异步阶段:分块 → 关键词提取 → 向量化 → 更新状态为 READY * * @param documents 解析后的文档列表 * @param title 文档标题 * @param sourceName 源文件名 * @param fileType 文件类型 * @param fileSize 文件大小 * @param content 原文内容(截断预览) * @param categoryId 分类ID * @param tags 标签列表 * @param chunkSize 分块大小(可选,覆盖全局配置) * @param overlap 重叠大小(可选,覆盖全局配置) * @return 创建完成的文档记录(status=PROCESSING) */ @Transactional(rollbackFor = Exception.class) public KnowledgeDocument uploadDocument(List documents, String title, String sourceName, String fileType, Long fileSize, String content, Long categoryId, List tags, Integer chunkSize, Integer overlap) { // 0. 内容去重检查 String contentHash = computeContentHash(content); if (contentHash != null) { String duplicateTitle = checkContentDuplicate(contentHash); if (duplicateTitle != null) { throw new RuntimeException("文档内容重复,已有同名文档: " + duplicateTitle); } } // 存储 per-doc 分块参数到 extraConfig(便于重新处理时复用) Map extraConfig = new HashMap<>(); if (chunkSize != null) extraConfig.put("chunkSize", chunkSize); if (overlap != null) extraConfig.put("overlap", overlap); // 1. 创建文档记录(状态 PROCESSING) KnowledgeDocument docRecord = KnowledgeDocument.builder() .title(title != null ? title : sourceName) .sourceName(sourceName) .fileType(fileType) .fileSize(fileSize != null ? fileSize : 0L) .content(content != null && content.length() > 2000 ? content.substring(0, 2000) : content) .categoryId(categoryId != null ? categoryId : 0L) .tags(tags != null ? Map.of("tags", tags) : null) .contentHash(contentHash) .enabled(true) .extraConfig(extraConfig.isEmpty() ? null : extraConfig) .status("PROCESSING") .chunkCount(0) .build(); documentMapper.insert(docRecord); // 2. 触发异步处理(分块 → 关键词 → 向量化 → 更新状态) documentProcessingService.processDocumentAsync( docRecord.getId(), documents, sourceName, title, categoryId, tags, chunkSize, overlap); log.info("文档已创建,后台处理中: id={}, title={}", docRecord.getId(), docRecord.getTitle()); return docRecord; } /** * 解析文件并上传 */ public KnowledgeDocument uploadFile(MultipartFile file, String title, Long categoryId, List tags, Integer chunkSize, Integer overlap) { List documents = tikaDocumentReader.read(file); String fileType = getFileExtension(file.getOriginalFilename()); return uploadDocument(documents, title != null ? title : file.getOriginalFilename(), file.getOriginalFilename(), fileType, file.getSize(), documents.get(0).getText(), categoryId, tags, chunkSize, overlap); } /** * 解析字符串并上传 */ public KnowledgeDocument uploadString(String content, String title, Long categoryId, List tags, Integer chunkSize, Integer overlap) { List documents = simpleStringDocumentReader.read(content); return uploadDocument(documents, title, title, "txt", (long) content.length(), content, categoryId, tags, chunkSize, overlap); } /** * 解析 Markdown 文件并上传 */ public KnowledgeDocument uploadMarkdown(MultipartFile file, String title, Long categoryId, List tags, Integer chunkSize, Integer overlap) { List documents = markdownDocumentLoader.loadMarkdownFromFile(file); String content = documents.stream().map(Document::getText).collect(Collectors.joining("\n")); return uploadDocument(documents, title != null ? title : file.getOriginalFilename(), file.getOriginalFilename(), "md", file.getSize(), content, categoryId, tags, chunkSize, overlap); } /** * 解析 JSON 文件(基本方式)并上传 */ public KnowledgeDocument uploadJsonBasic(MultipartFile file, String title, Long categoryId, List tags, Integer chunkSize, Integer overlap) { List documents = jsonDocumentLoader.loadBasicJson(file); String content = documents.stream().map(Document::getText).collect(Collectors.joining("\n")); return uploadDocument(documents, title != null ? title : file.getOriginalFilename(), file.getOriginalFilename(), "json", file.getSize(), content, categoryId, tags, chunkSize, overlap); } /** * 解析 JSON 文件(按字段)并上传 */ public KnowledgeDocument uploadJsonFields(MultipartFile file, List fields, String title, Long categoryId, List tags, Integer chunkSize, Integer overlap) { List documents = jsonDocumentLoader.loadJsonByFields(file, fields.toArray(new String[0])); String content = documents.stream().map(Document::getText).collect(Collectors.joining("\n")); return uploadDocument(documents, title != null ? title : file.getOriginalFilename(), file.getOriginalFilename(), "json", file.getSize(), content, categoryId, tags, chunkSize, overlap); } /** * 解析 JSON 文件(按指针)并上传 */ public KnowledgeDocument uploadJsonPointer(MultipartFile file, String pointer, String title, Long categoryId, List tags, Integer chunkSize, Integer overlap) { List documents = jsonDocumentLoader.loadJsonByPointer(file, pointer); String content = documents.stream().map(Document::getText).collect(Collectors.joining("\n")); return uploadDocument(documents, title != null ? title : file.getOriginalFilename(), file.getOriginalFilename(), "json", file.getSize(), content, categoryId, tags, chunkSize, overlap); } // ==================== 文档管理 ==================== /** * 分页查询文档列表(手动分页,支持关键词搜索 + 标签筛选) */ public Map listDocuments(int page, int size, Long categoryId, String status, String keyword, String tag) { // 参数安全校验 if (page < 1) page = 1; if (size < 1 || size > 100) size = 10; // LIKE 通配符转义 String kw = (keyword != null && !keyword.trim().isEmpty()) ? keyword.trim().replace("%", "\\%").replace("_", "\\_") : null; // 构建基础条件(用于 count 和 list) QueryWrapper countWrapper = new QueryWrapper<>(); if (categoryId != null && categoryId > 0) { countWrapper.eq("category_id", categoryId); } if (status != null && !status.isEmpty()) { countWrapper.eq("status", status); } if (kw != null) { countWrapper.and(w -> w.like("title", kw).or().like("source_name", kw)); } // P1-2.3: 标签筛选(JSONB 包含查询,使用 ObjectMapper 构建合法 JSON) if (tag != null && !tag.trim().isEmpty()) { try { ObjectMapper om = new ObjectMapper(); String jsonArray = om.writeValueAsString(List.of(tag.trim())); countWrapper.apply("tags->'tags' @> {0}::jsonb", jsonArray); } catch (Exception e) { log.warn("构建标签筛选 JSON 失败: tag={}", tag, e); } } // 先查询总数(不加 ORDER BY) Long total = documentMapper.selectCount(countWrapper); // 构建列表查询条件 QueryWrapper listWrapper = new QueryWrapper<>(); if (categoryId != null && categoryId > 0) { listWrapper.eq("category_id", categoryId); } if (status != null && !status.isEmpty()) { listWrapper.eq("status", status); } if (kw != null) { listWrapper.and(w -> w.like("title", kw).or().like("source_name", kw)); } if (tag != null && !tag.trim().isEmpty()) { try { ObjectMapper om = new ObjectMapper(); String jsonArray = om.writeValueAsString(List.of(tag.trim())); listWrapper.apply("tags->'tags' @> {0}::jsonb", jsonArray); } catch (Exception e) { log.warn("构建标签筛选 JSON 失败: tag={}", tag, e); } } listWrapper.orderByDesc("create_time"); listWrapper.last("LIMIT " + size + " OFFSET " + (page - 1) * size); List records = documentMapper.selectList(listWrapper); Map result = new HashMap<>(); result.put("records", records); result.put("total", total); result.put("page", page); result.put("size", size); result.put("pages", (total + size - 1) / size); return result; } /** * 获取文档详情 */ public KnowledgeDocument getDocumentDetail(Long id) { return documentMapper.selectById(id); } /** * 获取文档的所有分块 */ public List> getDocumentChunks(Long id) { String documentId = String.valueOf(id); String sql = "SELECT id::text as id, content, metadata, create_time FROM vector_store " + "WHERE metadata->>'documentId' = ? " + " OR metadata->>'document_id' = ? " + " OR metadata->>'docId' = ? " + "ORDER BY " + chunkOrderSql(); List> chunks = jdbcTemplate.queryForList(sql, documentId, documentId, documentId); if (!chunks.isEmpty()) { return chunks; } KnowledgeDocument doc = documentMapper.selectById(id); if (doc == null) { return chunks; } return getDocumentChunksByTitleFallback(doc); } private List> getDocumentChunksByTitleFallback(KnowledgeDocument doc) { List filters = new ArrayList<>(); List params = new ArrayList<>(); if (doc.getSourceName() != null && !doc.getSourceName().isBlank()) { filters.add("metadata->>'sourceName' = ?"); params.add(doc.getSourceName()); } if (doc.getTitle() != null && !doc.getTitle().isBlank()) { filters.add("metadata->>'title' = ?"); params.add(doc.getTitle()); } if (filters.isEmpty()) { return List.of(); } String sql = "SELECT id::text as id, content, metadata, create_time FROM vector_store " + "WHERE " + String.join(" OR ", filters) + " " + "ORDER BY " + chunkOrderSql(); if (doc.getChunkCount() != null && doc.getChunkCount() > 0) { sql += " LIMIT ?"; params.add(doc.getChunkCount()); } return jdbcTemplate.queryForList(sql, params.toArray()); } private String chunkOrderSql() { return "CASE WHEN metadata->>'chunkIndex' ~ '^[0-9]+$' " + "THEN (metadata->>'chunkIndex')::int ELSE 0 END ASC, create_time ASC, id::text ASC"; } /** * 删除文档(逻辑删除 + 级联删除向量) */ @Transactional(rollbackFor = Exception.class) public int deleteDocument(Long id) { KnowledgeDocument doc = documentMapper.selectById(id); if (doc == null) { throw new RuntimeException("文档不存在"); } // 删除关联的向量 int vectorCount = deleteVectorsByDocumentId(String.valueOf(id)); // 逻辑删除文档记录 documentMapper.deleteById(id); log.info("删除文档: id={}, title={}, 删除向量数={}", id, doc.getTitle(), vectorCount); return vectorCount; } /** * 批量删除文档 * * @param ids 文档ID列表 * @return 批量操作结果 */ public Map batchDeleteDocuments(List ids) { int successCount = 0; int failCount = 0; List> details = new ArrayList<>(); for (Long id : ids) { try { int vectorCount = deleteDocument(id); successCount++; details.add(Map.of("id", id.toString(), "success", true, "deletedVectors", vectorCount)); } catch (Exception e) { failCount++; details.add(Map.of("id", id.toString(), "success", false, "message", e.getMessage())); log.warn("批量删除文档失败: id={}", id, e); } } Map result = new HashMap<>(); result.put("successCount", successCount); result.put("failCount", failCount); result.put("details", details); return result; } /** * 批量重新处理文档 * * @param ids 文档ID列表 * @return 批量操作结果 */ public Map batchReprocessDocuments(List ids) { int successCount = 0; int failCount = 0; List> details = new ArrayList<>(); for (Long id : ids) { try { reprocessDocument(id); successCount++; details.add(Map.of("id", id.toString(), "success", true)); } catch (Exception e) { failCount++; details.add(Map.of("id", id.toString(), "success", false, "message", e.getMessage())); log.warn("批量重新处理文档失败: id={}", id, e); } } Map result = new HashMap<>(); result.put("successCount", successCount); result.put("failCount", failCount); result.put("details", details); return result; } /** * 重新处理文档(异步:先标记为 PROCESSING,后台重新分块 + 向量化) */ @Transactional(rollbackFor = Exception.class) public KnowledgeDocument reprocessDocument(Long id) { KnowledgeDocument doc = documentMapper.selectById(id); if (doc == null) { throw new RuntimeException("文档不存在"); } if (doc.getContent() == null || doc.getContent().isEmpty()) { throw new RuntimeException("文档无内容,无法重新处理"); } // 标记为 PROCESSING,清空旧状态 doc.setStatus("PROCESSING"); doc.setChunkCount(0); doc.setErrorMessage(null); documentMapper.updateById(doc); // 解析内容并触发异步处理 List documents = simpleStringDocumentReader.read(doc.getContent()); documentProcessingService.reprocessDocumentAsync(id, documents); log.info("文档重新处理已提交: id={}, title={}", doc.getId(), doc.getTitle()); return doc; } /** * 更新文档元信息 */ public void updateDocumentMetadata(Long id, String title, Long categoryId, List tags) { KnowledgeDocument doc = documentMapper.selectById(id); if (doc == null) { throw new RuntimeException("文档不存在"); } if (title != null && !title.isEmpty()) { doc.setTitle(title); } if (categoryId != null) { doc.setCategoryId(categoryId); } if (tags != null) { doc.setTags(Map.of("tags", tags)); } documentMapper.updateById(doc); // 同步更新 vector_store 中对应的 metadata // 注意:Spring AI 当前没有直接更新 metadata 的 API // 这里我们先更新文档记录,metadata 的同步留到后续优化 log.info("更新文档元信息: id={}, title={}", id, doc.getTitle()); } // ==================== 启用/禁用 ==================== /** * P1-2.1: 切换文档启用/禁用状态 * 禁用后不参与 RAG 检索但保留数据,同步更新 vector_store metadata * 使用原子翻转 SQL 避免并发竞态 */ public KnowledgeDocument toggleDocument(Long id) { // 原子翻转:直接在 DB 层 NOT enabled,避免先 SELECT 再 UPDATE 的竞态问题 int updated = jdbcTemplate.update( "UPDATE knowledge_document SET enabled = NOT enabled WHERE id = ? AND is_delete = false", id); if (updated == 0) { throw new RuntimeException("文档不存在或已被删除"); } // 查询最新状态 KnowledgeDocument doc = documentMapper.selectById(id); if (doc == null) { throw new RuntimeException("文档不存在"); } // 同步更新 vector_store 中的 enabled metadata syncVectorEnabledMetadata(String.valueOf(id), Boolean.TRUE.equals(doc.getEnabled())); log.info("文档状态切换: id={}, enabled={}", id, doc.getEnabled()); return doc; } /** * P1-2.1: 批量启用/禁用文档 */ public Map batchToggleDocuments(List ids, boolean enabled) { int successCount = 0; int failCount = 0; for (Long id : ids) { try { KnowledgeDocument doc = documentMapper.selectById(id); if (doc == null) { failCount++; continue; } doc.setEnabled(enabled); documentMapper.updateById(doc); syncVectorEnabledMetadata(String.valueOf(id), enabled); successCount++; } catch (Exception e) { failCount++; log.warn("批量切换文档状态失败: id={}", id, e); } } Map result = new HashMap<>(); result.put("successCount", successCount); result.put("failCount", failCount); return result; } /** * 同步更新 vector_store 中指定文档所有分块的 enabled metadata * 使用 JdbcTemplate 直接 UPDATE JSON 字段 * 注意:Spring AI 的 PgVectorStore 自动建表时 metadata 是 json 类型,不是 jsonb * 统一写入字符串类型 "true"/"false",与 searchDocuments 的过滤条件保持一致 */ private void syncVectorEnabledMetadata(String documentId, boolean enabled) { // 使用 to_jsonb(?::text) 将字符串转换为 JSONB 字符串值,与 searchDocuments 过滤条件一致 String sql = "UPDATE vector_store SET metadata = (jsonb_set(metadata::jsonb, '{enabled}', " + "to_jsonb(?::text)))::json WHERE metadata->>'documentId' = ?"; jdbcTemplate.update(sql, enabled ? "true" : "false", documentId); } // ==================== 批量移动分类 ==================== /** * P1-2.2: 批量移动文档到目标分类 * 仅更新 category_id,不影响向量数据 */ @Transactional(rollbackFor = Exception.class) public int batchMoveDocuments(List ids, Long categoryId) { Long targetCategoryId = categoryId != null ? categoryId : 0L; int updated = 0; for (Long id : ids) { KnowledgeDocument doc = documentMapper.selectById(id); if (doc != null) { doc.setCategoryId(targetCategoryId); documentMapper.updateById(doc); updated++; } } log.info("批量移动文档分类: ids={}, categoryId={}, 实际更新={}", ids, targetCategoryId, updated); return updated; } // ==================== 标签管理 ==================== /** * P1-2.3: 获取标签列表(从所有文档的 tags JSONB 聚合去重,含使用次数) * * @return [{tag: "标签名", count: 使用次数}] */ public List> getTagList() { String sql = "SELECT tag_value AS tag, COUNT(*) AS count " + "FROM knowledge_document, jsonb_array_elements(tags->'tags') AS tag_value " + "WHERE is_delete = false AND tags IS NOT NULL AND tags->'tags' IS NOT NULL " + "GROUP BY tag_value ORDER BY count DESC, tag_value ASC"; try { return jdbcTemplate.queryForList(sql); } catch (Exception e) { log.warn("获取标签列表失败", e); return List.of(); } } // ==================== 分块管理 ==================== /** * P1-2.5: 更新单个分块内容(同步更新 vector_store 并重新向量化) */ @Transactional(rollbackFor = Exception.class) public void updateChunk(Long docId, int chunkIndex, String newContent) { KnowledgeDocument doc = documentMapper.selectById(docId); if (doc == null) { throw new RuntimeException("文档不存在"); } String docIdStr = String.valueOf(docId); // 查找对应的向量记录 ID String selectSql = "SELECT id::text FROM vector_store " + "WHERE (metadata->>'documentId' = ? OR metadata->>'document_id' = ?) " + "AND metadata->>'chunkIndex' = ? " + "LIMIT 1"; List vectorIds = jdbcTemplate.queryForList(selectSql, String.class, docIdStr, docIdStr, String.valueOf(chunkIndex)); if (vectorIds.isEmpty()) { throw new RuntimeException("未找到对应的分块向量记录"); } String vectorId = vectorIds.get(0); // 删除旧向量 pgVectorVectorStore.delete(List.of(vectorId)); // 构建新文档并重新向量化 Map meta = new HashMap<>(); meta.put("documentId", docIdStr); meta.put("chunkIndex", chunkIndex); meta.put("sourceName", doc.getSourceName()); meta.put("title", doc.getTitle()); if (doc.getCategoryId() != null && doc.getCategoryId() > 0) { meta.put("categoryId", String.valueOf(doc.getCategoryId())); } if (doc.getTags() != null && doc.getTags().containsKey("tags")) { meta.put("tags", doc.getTags().get("tags")); } // 标记启用状态,与文档当前状态保持一致 meta.put("enabled", String.valueOf(Boolean.TRUE.equals(doc.getEnabled()))); Document newDoc = new Document(vectorId, newContent, meta); // 关键词提取 List enriched = myKeywordEnricher.enrichDocuments(List.of(newDoc)); pgVectorVectorStore.add(enriched); log.info("更新分块: docId={}, chunkIndex={}, vectorId={}", docId, chunkIndex, vectorId); } /** * P1-2.5: 删除单个分块(同步从 vector_store 移除向量) */ @Transactional(rollbackFor = Exception.class) public void deleteChunk(Long docId, int chunkIndex) { KnowledgeDocument doc = documentMapper.selectById(docId); if (doc == null) { throw new RuntimeException("文档不存在"); } String docIdStr = String.valueOf(docId); // 查找对应的向量记录 ID String selectSql = "SELECT id::text FROM vector_store " + "WHERE (metadata->>'documentId' = ? OR metadata->>'document_id' = ?) " + "AND metadata->>'chunkIndex' = ?"; List vectorIds = jdbcTemplate.queryForList(selectSql, String.class, docIdStr, docIdStr, String.valueOf(chunkIndex)); if (!vectorIds.isEmpty()) { pgVectorVectorStore.delete(vectorIds); } // 更新文档的分块计数 doc.setChunkCount(Math.max(0, (doc.getChunkCount() != null ? doc.getChunkCount() : 1) - vectorIds.size())); documentMapper.updateById(doc); log.info("删除分块: docId={}, chunkIndex={}, 删除向量数={}", docId, chunkIndex, vectorIds.size()); } // ==================== 语义搜索 ==================== /** * 语义搜索 */ public List searchDocuments(String query, int topK, double similarityThreshold, Long categoryId) { List categoryIds = categoryId != null ? List.of(categoryId) : Collections.emptyList(); return searchDocuments(query, topK, similarityThreshold, categoryIds); } /** * 语义搜索(支持多个知识库分类过滤) */ public List searchDocuments(String query, int topK, double similarityThreshold, List categoryIds) { SearchRequest.Builder searchBuilder = SearchRequest.builder() .query(query) .topK(topK) .similarityThreshold(similarityThreshold); FilterExpressionBuilder builder = new FilterExpressionBuilder(); // P1-2.1: 始终过滤禁用的文档,只检索启用的 var enabledFilter = builder.eq("enabled", "true"); List categoryIdStrings = normalizeCategoryIds(categoryIds); if (!categoryIdStrings.isEmpty()) { List values = categoryIdStrings.stream() .map(value -> (Object) value) .collect(Collectors.toList()); var categoryFilter = builder.in("categoryId", values); searchBuilder.filterExpression(builder.and(enabledFilter, categoryFilter).build()); } else { searchBuilder.filterExpression(enabledFilter.build()); } List results; try { results = pgVectorVectorStore.similaritySearch(searchBuilder.build()); } catch (Exception e) { if (categoryIdStrings.isEmpty()) { throw e; } log.warn("向量检索分类过滤失败,改用本地 metadata 过滤兜底: categoryIds={}", categoryIdStrings, e); results = pgVectorVectorStore.similaritySearch(SearchRequest.builder() .query(query) .topK(Math.max(topK * 5, 20)) .similarityThreshold(similarityThreshold) .build()) .stream() .filter(doc -> categoryIdStrings.contains(getStringFromMetadata(doc.getMetadata(), "categoryId"))) .filter(doc -> "true".equals(getStringFromMetadata(doc.getMetadata(), "enabled"))) .limit(topK) .collect(Collectors.toList()); } List searchResults = new ArrayList<>(); for (Document doc : results) { Map metadata = doc.getMetadata(); SearchResult result = SearchResult.builder() .id(doc.getId()) .content(doc.getText()) .score(metadata.containsKey("distance") ? ((Number) metadata.get("distance")).doubleValue() : null) .sourceName(getStringFromMetadata(metadata, "sourceName")) .title(getStringFromMetadata(metadata, "title")) .chunkIndex(getIntegerFromMetadata(metadata, "chunkIndex")) .documentId(getStringFromMetadata(metadata, "documentId")) .metadata(metadata) .build(); searchResults.add(result); } return searchResults; } /** * 多模式搜索(P0-001: 支持向量/关键词/混合检索) * * @param searchMode 检索模式:VECTOR(默认)/ KEYWORD / HYBRID */ public List searchDocuments(String query, int topK, double similarityThreshold, List categoryIds, String searchMode) { if (searchMode == null || searchMode.isEmpty() || "VECTOR".equalsIgnoreCase(searchMode)) { // 默认向量检索,向后兼容 List results = searchDocuments(query, topK, similarityThreshold, categoryIds); results.forEach(r -> r.setSearchMode("VECTOR")); return results; } // 委托给 HybridSearchService 处理 KEYWORD 和 HYBRID 模式 if (hybridSearchService != null) { com.wok.supportbot.rag.SearchMode mode; try { mode = com.wok.supportbot.rag.SearchMode.valueOf(searchMode.toUpperCase()); } catch (IllegalArgumentException e) { mode = com.wok.supportbot.rag.SearchMode.VECTOR; } return hybridSearchService.search(query, mode, topK, similarityThreshold, categoryIds); } // HybridSearchService 不可用时降级到向量检索 log.warn("HybridSearchService 不可用,降级为向量检索"); return searchDocuments(query, topK, similarityThreshold, categoryIds); } @Autowired(required = false) private com.wok.supportbot.rag.HybridSearchService hybridSearchService; private List normalizeCategoryIds(List categoryIds) { if (categoryIds == null || categoryIds.isEmpty()) { return Collections.emptyList(); } return categoryIds.stream() .filter(Objects::nonNull) .filter(id -> id > 0) .map(String::valueOf) .distinct() .collect(Collectors.toList()); } // ==================== 统计 ==================== /** * 获取知识库统计信息 */ public Map getStats() { // 文档统计 Long totalDocuments = documentMapper.selectCount(null); // 按文件类型统计 String typeSql = "SELECT file_type, COUNT(*) as count FROM knowledge_document WHERE is_delete = false GROUP BY file_type"; List> typeStats = jdbcTemplate.queryForList(typeSql); Map byFileType = typeStats.stream() .collect(Collectors.toMap( r -> (String) r.get("file_type"), r -> ((Number) r.get("count")).longValue() )); // 按分类统计 String catSql = "SELECT c.name, COUNT(d.id) as count FROM knowledge_document d " + "LEFT JOIN knowledge_category c ON d.category_id = c.id " + "WHERE d.is_delete = false GROUP BY c.name"; List> catStats; try { catStats = jdbcTemplate.queryForList(catSql); } catch (Exception e) { catStats = new ArrayList<>(); } // 向量总数 String vectorSql = "SELECT COUNT(*) FROM vector_store"; Long totalVectors; try { totalVectors = jdbcTemplate.queryForObject(vectorSql, Long.class); } catch (Exception e) { totalVectors = 0L; } // 最近上传时间 String lastUploadSql = "SELECT MAX(create_time) FROM knowledge_document WHERE is_delete = false"; Date lastUploadTime = jdbcTemplate.queryForObject(lastUploadSql, Date.class); Map stats = new LinkedHashMap<>(); stats.put("totalDocuments", totalDocuments); stats.put("totalVectors", totalVectors); stats.put("lastUploadTime", lastUploadTime); stats.put("byFileType", byFileType); stats.put("byCategory", catStats); return stats; } // ==================== 分类管理 ==================== /** * 获取分类树 */ public List getCategoryTree() { List categories = categoryMapper.selectList( new QueryWrapper().orderByAsc("sort_order")); Map nodeMap = new LinkedHashMap<>(); List rootNodes = new ArrayList<>(); for (KnowledgeCategory cat : categories) { CategoryNode node = CategoryNode.builder() .id(cat.getId()) .name(cat.getName()) .description(cat.getDescription()) .parentId(cat.getParentId()) .sortOrder(cat.getSortOrder()) .documentCount(cat.getDocumentCount()) .children(new ArrayList<>()) .build(); nodeMap.put(cat.getId(), node); } for (CategoryNode node : nodeMap.values()) { if (node.getParentId() == null || node.getParentId() == 0) { rootNodes.add(node); } else { CategoryNode parent = nodeMap.get(node.getParentId()); if (parent != null) { parent.getChildren().add(node); } else { rootNodes.add(node); } } } return rootNodes; } /** * 获取分类列表 */ public List listCategories() { return categoryMapper.selectList( new QueryWrapper().orderByAsc("sort_order")); } /** * 创建分类 */ public KnowledgeCategory createCategory(String name, String description, Long parentId, Integer sortOrder) { KnowledgeCategory category = KnowledgeCategory.builder() .name(name) .description(description) .parentId(parentId != null ? parentId : 0L) .sortOrder(sortOrder != null ? sortOrder : 0) .documentCount(0) .build(); categoryMapper.insert(category); return category; } /** * 更新分类 */ public void updateCategory(Long id, String name, String description, Integer sortOrder) { KnowledgeCategory category = categoryMapper.selectById(id); if (category == null) { throw new RuntimeException("分类不存在"); } if (name != null && !name.isEmpty()) { category.setName(name); } if (description != null) { category.setDescription(description); } if (sortOrder != null) { category.setSortOrder(sortOrder); } categoryMapper.updateById(category); } /** * 删除分类(不删除文档,仅清空关联) */ @Transactional(rollbackFor = Exception.class) public void deleteCategory(Long id) { // 将关联的文档 category_id 设为 0 KnowledgeDocument updateDoc = new KnowledgeDocument(); updateDoc.setCategoryId(0L); documentMapper.update(updateDoc, new QueryWrapper().eq("category_id", id)); // 逻辑删除分类 categoryMapper.deleteById(id); } // ==================== 内部方法 ==================== /** * 计算文本内容的 SHA-256 哈希值 */ private String computeContentHash(String content) { if (content == null || content.isEmpty()) { return null; } try { MessageDigest digest = MessageDigest.getInstance("SHA-256"); byte[] hashBytes = digest.digest(content.getBytes(StandardCharsets.UTF_8)); StringBuilder sb = new StringBuilder(); for (byte b : hashBytes) { sb.append(String.format("%02x", b)); } return sb.toString(); } catch (NoSuchAlgorithmException e) { log.error("SHA-256 算法不可用", e); return null; } } /** * 检查内容是否重复 * @param contentHash 内容哈希值 * @return 重复文档的标题,如果不存在重复则返回 null */ private String checkContentDuplicate(String contentHash) { if (contentHash == null) { return null; } QueryWrapper wrapper = new QueryWrapper<>(); wrapper.eq("content_hash", contentHash); wrapper.select("title"); KnowledgeDocument existing = documentMapper.selectOne(wrapper); return existing != null ? existing.getTitle() : null; } /** * 根据文档ID删除 vector_store 中关联的所有向量 */ private int deleteVectorsByDocumentId(String documentId) { String sql = "SELECT id::text FROM vector_store WHERE metadata->>'documentId' = ?"; List ids = jdbcTemplate.queryForList(sql, String.class, documentId); if (!ids.isEmpty()) { pgVectorVectorStore.delete(ids); log.debug("删除向量: documentId={}, count={}", documentId, ids.size()); } return ids.size(); } /** * 获取文件扩展名 */ private String getFileExtension(String filename) { if (filename == null || !filename.contains(".")) { return "unknown"; } return filename.substring(filename.lastIndexOf(".") + 1).toLowerCase(); } /** * 从 metadata 中安全获取字符串值 */ private String getStringFromMetadata(Map metadata, String key) { Object value = metadata.get(key); return value != null ? value.toString() : null; } /** * 从 metadata 中安全获取整数值 */ private Integer getIntegerFromMetadata(Map metadata, String key) { Object value = metadata.get(key); if (value == null) return null; if (value instanceof Number) { return ((Number) value).intValue(); } try { return Integer.parseInt(value.toString()); } catch (NumberFormatException e) { return null; } } }