You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
1044 lines
41 KiB
1044 lines
41 KiB
package com.wok.supportbot.service;
|
|
|
|
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
|
|
import com.wok.supportbot.dao.KnowledgeCategoryMapper;
|
|
import com.wok.supportbot.dao.KnowledgeDocumentMapper;
|
|
import com.wok.supportbot.document.extract.JsonDocumentLoader;
|
|
import com.wok.supportbot.document.extract.MarkdownDocumentLoader;
|
|
import com.wok.supportbot.document.extract.SimpleStringDocumentReader;
|
|
import com.wok.supportbot.document.extract.TikaDocumentReader;
|
|
import com.wok.supportbot.document.transform.MyKeywordEnricher;
|
|
import com.wok.supportbot.document.transform.MyTokenTextSplitter;
|
|
import com.wok.supportbot.entity.CategoryNode;
|
|
import com.wok.supportbot.entity.KnowledgeCategory;
|
|
import com.wok.supportbot.entity.KnowledgeDocument;
|
|
import com.wok.supportbot.entity.SearchResult;
|
|
import lombok.extern.slf4j.Slf4j;
|
|
import org.springframework.ai.document.Document;
|
|
import org.springframework.ai.vectorstore.SearchRequest;
|
|
import org.springframework.ai.vectorstore.VectorStore;
|
|
import org.springframework.ai.vectorstore.filter.FilterExpressionBuilder;
|
|
import org.springframework.beans.factory.annotation.Autowired;
|
|
import org.springframework.jdbc.core.JdbcTemplate;
|
|
import org.springframework.stereotype.Service;
|
|
import org.springframework.transaction.annotation.Transactional;
|
|
import org.springframework.web.multipart.MultipartFile;
|
|
|
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
|
|
|
import java.nio.charset.StandardCharsets;
|
|
import java.security.MessageDigest;
|
|
import java.security.NoSuchAlgorithmException;
|
|
import java.util.*;
|
|
import java.util.stream.Collectors;
|
|
|
|
/**
|
|
* 知识库文档管理服务
|
|
* 统一管理文档的上传、删除、搜索、统计、分类等操作
|
|
*/
|
|
@Service
|
|
@Slf4j
|
|
public class DocumentService {
|
|
|
|
@Autowired
|
|
private KnowledgeDocumentMapper documentMapper;
|
|
|
|
@Autowired
|
|
private KnowledgeCategoryMapper categoryMapper;
|
|
|
|
@Autowired
|
|
private JdbcTemplate jdbcTemplate;
|
|
|
|
@Autowired
|
|
private VectorStore pgVectorVectorStore;
|
|
|
|
@Autowired
|
|
private MyTokenTextSplitter myTokenTextSplitter;
|
|
|
|
@Autowired
|
|
private MyKeywordEnricher myKeywordEnricher;
|
|
|
|
@Autowired
|
|
private TikaDocumentReader tikaDocumentReader;
|
|
|
|
@Autowired
|
|
private SimpleStringDocumentReader simpleStringDocumentReader;
|
|
|
|
@Autowired
|
|
private MarkdownDocumentLoader markdownDocumentLoader;
|
|
|
|
@Autowired
|
|
private JsonDocumentLoader jsonDocumentLoader;
|
|
|
|
@Autowired
|
|
private DocumentProcessingService documentProcessingService;
|
|
|
|
// ==================== 文档上传 ====================
|
|
|
|
/**
|
|
* 统一文档上传流程：创建记录（PROCESSING）→ 触发异步处理
|
|
* 同步阶段：内容去重检查 + 创建 PROCESSING 记录
|
|
* 异步阶段：分块 → 关键词提取 → 向量化 → 更新状态为 READY
|
|
*
|
|
* @param documents 解析后的文档列表
|
|
* @param title 文档标题
|
|
* @param sourceName 源文件名
|
|
* @param fileType 文件类型
|
|
* @param fileSize 文件大小
|
|
* @param content 原文内容（截断预览）
|
|
* @param categoryId 分类ID
|
|
* @param tags 标签列表
|
|
* @param chunkSize 分块大小（可选，覆盖全局配置）
|
|
* @param overlap 重叠大小（可选，覆盖全局配置）
|
|
* @return 创建完成的文档记录（status=PROCESSING）
|
|
*/
|
|
@Transactional(rollbackFor = Exception.class)
|
|
public KnowledgeDocument uploadDocument(List<Document> documents, String title, String sourceName,
|
|
String fileType, Long fileSize, String content,
|
|
Long categoryId, List<String> tags,
|
|
Integer chunkSize, Integer overlap) {
|
|
// 0. 内容去重检查
|
|
String contentHash = computeContentHash(content);
|
|
if (contentHash != null) {
|
|
String duplicateTitle = checkContentDuplicate(contentHash);
|
|
if (duplicateTitle != null) {
|
|
throw new RuntimeException("文档内容重复，已有同名文档: " + duplicateTitle);
|
|
}
|
|
}
|
|
|
|
// 存储 per-doc 分块参数到 extraConfig（便于重新处理时复用）
|
|
Map<String, Object> extraConfig = new HashMap<>();
|
|
if (chunkSize != null) extraConfig.put("chunkSize", chunkSize);
|
|
if (overlap != null) extraConfig.put("overlap", overlap);
|
|
|
|
// 1. 创建文档记录（状态 PROCESSING）
|
|
KnowledgeDocument docRecord = KnowledgeDocument.builder()
|
|
.title(title != null ? title : sourceName)
|
|
.sourceName(sourceName)
|
|
.fileType(fileType)
|
|
.fileSize(fileSize != null ? fileSize : 0L)
|
|
.content(content != null && content.length() > 2000 ? content.substring(0, 2000) : content)
|
|
.categoryId(categoryId != null ? categoryId : 0L)
|
|
.tags(tags != null ? Map.of("tags", tags) : null)
|
|
.contentHash(contentHash)
|
|
.enabled(true)
|
|
.extraConfig(extraConfig.isEmpty() ? null : extraConfig)
|
|
.status("PROCESSING")
|
|
.chunkCount(0)
|
|
.build();
|
|
documentMapper.insert(docRecord);
|
|
|
|
// 2. 触发异步处理（分块 → 关键词 → 向量化 → 更新状态）
|
|
documentProcessingService.processDocumentAsync(
|
|
docRecord.getId(), documents, sourceName, title, categoryId, tags,
|
|
chunkSize, overlap);
|
|
|
|
log.info("文档已创建，后台处理中: id={}, title={}", docRecord.getId(), docRecord.getTitle());
|
|
return docRecord;
|
|
}
|
|
|
|
/**
|
|
* 解析文件并上传
|
|
*/
|
|
public KnowledgeDocument uploadFile(MultipartFile file, String title, Long categoryId, List<String> tags,
|
|
Integer chunkSize, Integer overlap) {
|
|
List<Document> documents = tikaDocumentReader.read(file);
|
|
String fileType = getFileExtension(file.getOriginalFilename());
|
|
return uploadDocument(documents,
|
|
title != null ? title : file.getOriginalFilename(),
|
|
file.getOriginalFilename(),
|
|
fileType,
|
|
file.getSize(),
|
|
documents.get(0).getText(),
|
|
categoryId,
|
|
tags, chunkSize, overlap);
|
|
}
|
|
|
|
/**
|
|
* 解析字符串并上传
|
|
*/
|
|
public KnowledgeDocument uploadString(String content, String title, Long categoryId, List<String> tags,
|
|
Integer chunkSize, Integer overlap) {
|
|
List<Document> documents = simpleStringDocumentReader.read(content);
|
|
return uploadDocument(documents, title, title, "txt",
|
|
(long) content.length(), content, categoryId, tags, chunkSize, overlap);
|
|
}
|
|
|
|
/**
|
|
* 解析 Markdown 文件并上传
|
|
*/
|
|
public KnowledgeDocument uploadMarkdown(MultipartFile file, String title, Long categoryId, List<String> tags,
|
|
Integer chunkSize, Integer overlap) {
|
|
List<Document> documents = markdownDocumentLoader.loadMarkdownFromFile(file);
|
|
String content = documents.stream().map(Document::getText).collect(Collectors.joining("\n"));
|
|
return uploadDocument(documents,
|
|
title != null ? title : file.getOriginalFilename(),
|
|
file.getOriginalFilename(),
|
|
"md",
|
|
file.getSize(),
|
|
content,
|
|
categoryId,
|
|
tags, chunkSize, overlap);
|
|
}
|
|
|
|
/**
|
|
* 解析 JSON 文件（基本方式）并上传
|
|
*/
|
|
public KnowledgeDocument uploadJsonBasic(MultipartFile file, String title, Long categoryId, List<String> tags,
|
|
Integer chunkSize, Integer overlap) {
|
|
List<Document> documents = jsonDocumentLoader.loadBasicJson(file);
|
|
String content = documents.stream().map(Document::getText).collect(Collectors.joining("\n"));
|
|
return uploadDocument(documents,
|
|
title != null ? title : file.getOriginalFilename(),
|
|
file.getOriginalFilename(),
|
|
"json",
|
|
file.getSize(),
|
|
content,
|
|
categoryId,
|
|
tags, chunkSize, overlap);
|
|
}
|
|
|
|
/**
|
|
* 解析 JSON 文件（按字段）并上传
|
|
*/
|
|
public KnowledgeDocument uploadJsonFields(MultipartFile file, List<String> fields, String title,
|
|
Long categoryId, List<String> tags,
|
|
Integer chunkSize, Integer overlap) {
|
|
List<Document> documents = jsonDocumentLoader.loadJsonByFields(file, fields.toArray(new String[0]));
|
|
String content = documents.stream().map(Document::getText).collect(Collectors.joining("\n"));
|
|
return uploadDocument(documents,
|
|
title != null ? title : file.getOriginalFilename(),
|
|
file.getOriginalFilename(),
|
|
"json",
|
|
file.getSize(),
|
|
content,
|
|
categoryId,
|
|
tags, chunkSize, overlap);
|
|
}
|
|
|
|
/**
|
|
* 解析 JSON 文件（按指针）并上传
|
|
*/
|
|
public KnowledgeDocument uploadJsonPointer(MultipartFile file, String pointer, String title,
|
|
Long categoryId, List<String> tags,
|
|
Integer chunkSize, Integer overlap) {
|
|
List<Document> documents = jsonDocumentLoader.loadJsonByPointer(file, pointer);
|
|
String content = documents.stream().map(Document::getText).collect(Collectors.joining("\n"));
|
|
return uploadDocument(documents,
|
|
title != null ? title : file.getOriginalFilename(),
|
|
file.getOriginalFilename(),
|
|
"json",
|
|
file.getSize(),
|
|
content,
|
|
categoryId,
|
|
tags, chunkSize, overlap);
|
|
}
|
|
|
|
// ==================== 文档管理 ====================
|
|
|
|
/**
|
|
* 分页查询文档列表（手动分页，支持关键词搜索 + 标签筛选）
|
|
*/
|
|
public Map<String, Object> listDocuments(int page, int size, Long categoryId, String status, String keyword, String tag) {
|
|
// 参数安全校验
|
|
if (page < 1) page = 1;
|
|
if (size < 1 || size > 100) size = 10;
|
|
// LIKE 通配符转义
|
|
String kw = (keyword != null && !keyword.trim().isEmpty())
|
|
? keyword.trim().replace("%", "\\%").replace("_", "\\_") : null;
|
|
|
|
// 构建基础条件（用于 count 和 list）
|
|
QueryWrapper<KnowledgeDocument> countWrapper = new QueryWrapper<>();
|
|
if (categoryId != null && categoryId > 0) {
|
|
countWrapper.eq("category_id", categoryId);
|
|
}
|
|
if (status != null && !status.isEmpty()) {
|
|
countWrapper.eq("status", status);
|
|
}
|
|
if (kw != null) {
|
|
countWrapper.and(w -> w.like("title", kw).or().like("source_name", kw));
|
|
}
|
|
// P1-2.3: 标签筛选（JSONB 包含查询，使用 ObjectMapper 构建合法 JSON）
|
|
if (tag != null && !tag.trim().isEmpty()) {
|
|
try {
|
|
ObjectMapper om = new ObjectMapper();
|
|
String jsonArray = om.writeValueAsString(List.of(tag.trim()));
|
|
countWrapper.apply("tags->'tags' @> {0}::jsonb", jsonArray);
|
|
} catch (Exception e) {
|
|
log.warn("构建标签筛选 JSON 失败: tag={}", tag, e);
|
|
}
|
|
}
|
|
|
|
// 先查询总数（不加 ORDER BY）
|
|
Long total = documentMapper.selectCount(countWrapper);
|
|
|
|
// 构建列表查询条件
|
|
QueryWrapper<KnowledgeDocument> listWrapper = new QueryWrapper<>();
|
|
if (categoryId != null && categoryId > 0) {
|
|
listWrapper.eq("category_id", categoryId);
|
|
}
|
|
if (status != null && !status.isEmpty()) {
|
|
listWrapper.eq("status", status);
|
|
}
|
|
if (kw != null) {
|
|
listWrapper.and(w -> w.like("title", kw).or().like("source_name", kw));
|
|
}
|
|
if (tag != null && !tag.trim().isEmpty()) {
|
|
try {
|
|
ObjectMapper om = new ObjectMapper();
|
|
String jsonArray = om.writeValueAsString(List.of(tag.trim()));
|
|
listWrapper.apply("tags->'tags' @> {0}::jsonb", jsonArray);
|
|
} catch (Exception e) {
|
|
log.warn("构建标签筛选 JSON 失败: tag={}", tag, e);
|
|
}
|
|
}
|
|
listWrapper.orderByDesc("create_time");
|
|
listWrapper.last("LIMIT " + size + " OFFSET " + (page - 1) * size);
|
|
List<KnowledgeDocument> records = documentMapper.selectList(listWrapper);
|
|
|
|
Map<String, Object> result = new HashMap<>();
|
|
result.put("records", records);
|
|
result.put("total", total);
|
|
result.put("page", page);
|
|
result.put("size", size);
|
|
result.put("pages", (total + size - 1) / size);
|
|
return result;
|
|
}
|
|
|
|
/**
|
|
* 获取文档详情
|
|
*/
|
|
public KnowledgeDocument getDocumentDetail(Long id) {
|
|
return documentMapper.selectById(id);
|
|
}
|
|
|
|
/**
|
|
* 获取文档的所有分块
|
|
*/
|
|
public List<Map<String, Object>> getDocumentChunks(Long id) {
|
|
String documentId = String.valueOf(id);
|
|
String sql = "SELECT id::text as id, content, metadata, create_time FROM vector_store " +
|
|
"WHERE metadata->>'documentId' = ? " +
|
|
" OR metadata->>'document_id' = ? " +
|
|
" OR metadata->>'docId' = ? " +
|
|
"ORDER BY " + chunkOrderSql();
|
|
List<Map<String, Object>> chunks = jdbcTemplate.queryForList(sql, documentId, documentId, documentId);
|
|
if (!chunks.isEmpty()) {
|
|
return chunks;
|
|
}
|
|
|
|
KnowledgeDocument doc = documentMapper.selectById(id);
|
|
if (doc == null) {
|
|
return chunks;
|
|
}
|
|
return getDocumentChunksByTitleFallback(doc);
|
|
}
|
|
|
|
private List<Map<String, Object>> getDocumentChunksByTitleFallback(KnowledgeDocument doc) {
|
|
List<String> filters = new ArrayList<>();
|
|
List<Object> params = new ArrayList<>();
|
|
if (doc.getSourceName() != null && !doc.getSourceName().isBlank()) {
|
|
filters.add("metadata->>'sourceName' = ?");
|
|
params.add(doc.getSourceName());
|
|
}
|
|
if (doc.getTitle() != null && !doc.getTitle().isBlank()) {
|
|
filters.add("metadata->>'title' = ?");
|
|
params.add(doc.getTitle());
|
|
}
|
|
if (filters.isEmpty()) {
|
|
return List.of();
|
|
}
|
|
|
|
String sql = "SELECT id::text as id, content, metadata, create_time FROM vector_store " +
|
|
"WHERE " + String.join(" OR ", filters) + " " +
|
|
"ORDER BY " + chunkOrderSql();
|
|
if (doc.getChunkCount() != null && doc.getChunkCount() > 0) {
|
|
sql += " LIMIT ?";
|
|
params.add(doc.getChunkCount());
|
|
}
|
|
return jdbcTemplate.queryForList(sql, params.toArray());
|
|
}
|
|
|
|
private String chunkOrderSql() {
|
|
return "CASE WHEN metadata->>'chunkIndex' ~ '^[0-9]+$' " +
|
|
"THEN (metadata->>'chunkIndex')::int ELSE 0 END ASC, create_time ASC, id::text ASC";
|
|
}
|
|
|
|
/**
|
|
* 删除文档（逻辑删除 + 级联删除向量）
|
|
*/
|
|
@Transactional(rollbackFor = Exception.class)
|
|
public int deleteDocument(Long id) {
|
|
KnowledgeDocument doc = documentMapper.selectById(id);
|
|
if (doc == null) {
|
|
throw new RuntimeException("文档不存在");
|
|
}
|
|
// 删除关联的向量
|
|
int vectorCount = deleteVectorsByDocumentId(String.valueOf(id));
|
|
// 逻辑删除文档记录
|
|
documentMapper.deleteById(id);
|
|
log.info("删除文档: id={}, title={}, 删除向量数={}", id, doc.getTitle(), vectorCount);
|
|
return vectorCount;
|
|
}
|
|
|
|
/**
|
|
* 批量删除文档
|
|
*
|
|
* @param ids 文档ID列表
|
|
* @return 批量操作结果
|
|
*/
|
|
public Map<String, Object> batchDeleteDocuments(List<Long> ids) {
|
|
int successCount = 0;
|
|
int failCount = 0;
|
|
List<Map<String, Object>> details = new ArrayList<>();
|
|
|
|
for (Long id : ids) {
|
|
try {
|
|
int vectorCount = deleteDocument(id);
|
|
successCount++;
|
|
details.add(Map.of("id", id.toString(), "success", true, "deletedVectors", vectorCount));
|
|
} catch (Exception e) {
|
|
failCount++;
|
|
details.add(Map.of("id", id.toString(), "success", false, "message", e.getMessage()));
|
|
log.warn("批量删除文档失败: id={}", id, e);
|
|
}
|
|
}
|
|
|
|
Map<String, Object> result = new HashMap<>();
|
|
result.put("successCount", successCount);
|
|
result.put("failCount", failCount);
|
|
result.put("details", details);
|
|
return result;
|
|
}
|
|
|
|
/**
|
|
* 批量重新处理文档
|
|
*
|
|
* @param ids 文档ID列表
|
|
* @return 批量操作结果
|
|
*/
|
|
public Map<String, Object> batchReprocessDocuments(List<Long> ids) {
|
|
int successCount = 0;
|
|
int failCount = 0;
|
|
List<Map<String, Object>> details = new ArrayList<>();
|
|
|
|
for (Long id : ids) {
|
|
try {
|
|
reprocessDocument(id);
|
|
successCount++;
|
|
details.add(Map.of("id", id.toString(), "success", true));
|
|
} catch (Exception e) {
|
|
failCount++;
|
|
details.add(Map.of("id", id.toString(), "success", false, "message", e.getMessage()));
|
|
log.warn("批量重新处理文档失败: id={}", id, e);
|
|
}
|
|
}
|
|
|
|
Map<String, Object> result = new HashMap<>();
|
|
result.put("successCount", successCount);
|
|
result.put("failCount", failCount);
|
|
result.put("details", details);
|
|
return result;
|
|
}
|
|
|
|
/**
|
|
* 重新处理文档（异步：先标记为 PROCESSING，后台重新分块 + 向量化）
|
|
*/
|
|
@Transactional(rollbackFor = Exception.class)
|
|
public KnowledgeDocument reprocessDocument(Long id) {
|
|
KnowledgeDocument doc = documentMapper.selectById(id);
|
|
if (doc == null) {
|
|
throw new RuntimeException("文档不存在");
|
|
}
|
|
if (doc.getContent() == null || doc.getContent().isEmpty()) {
|
|
throw new RuntimeException("文档无内容，无法重新处理");
|
|
}
|
|
|
|
// 标记为 PROCESSING，清空旧状态
|
|
doc.setStatus("PROCESSING");
|
|
doc.setChunkCount(0);
|
|
doc.setErrorMessage(null);
|
|
documentMapper.updateById(doc);
|
|
|
|
// 解析内容并触发异步处理
|
|
List<Document> documents = simpleStringDocumentReader.read(doc.getContent());
|
|
documentProcessingService.reprocessDocumentAsync(id, documents);
|
|
|
|
log.info("文档重新处理已提交: id={}, title={}", doc.getId(), doc.getTitle());
|
|
return doc;
|
|
}
|
|
|
|
/**
|
|
* 更新文档元信息
|
|
*/
|
|
public void updateDocumentMetadata(Long id, String title, Long categoryId, List<String> tags) {
|
|
KnowledgeDocument doc = documentMapper.selectById(id);
|
|
if (doc == null) {
|
|
throw new RuntimeException("文档不存在");
|
|
}
|
|
if (title != null && !title.isEmpty()) {
|
|
doc.setTitle(title);
|
|
}
|
|
if (categoryId != null) {
|
|
doc.setCategoryId(categoryId);
|
|
}
|
|
if (tags != null) {
|
|
doc.setTags(Map.of("tags", tags));
|
|
}
|
|
documentMapper.updateById(doc);
|
|
|
|
// 同步更新 vector_store 中对应的 metadata
|
|
// 注意：Spring AI 当前没有直接更新 metadata 的 API
|
|
// 这里我们先更新文档记录，metadata 的同步留到后续优化
|
|
log.info("更新文档元信息: id={}, title={}", id, doc.getTitle());
|
|
}
|
|
|
|
// ==================== 启用/禁用 ====================
|
|
|
|
/**
|
|
* P1-2.1: 切换文档启用/禁用状态
|
|
* 禁用后不参与 RAG 检索但保留数据，同步更新 vector_store metadata
|
|
* 使用原子翻转 SQL 避免并发竞态
|
|
*/
|
|
public KnowledgeDocument toggleDocument(Long id) {
|
|
// 原子翻转：直接在 DB 层 NOT enabled，避免先 SELECT 再 UPDATE 的竞态问题
|
|
int updated = jdbcTemplate.update(
|
|
"UPDATE knowledge_document SET enabled = NOT enabled WHERE id = ? AND is_delete = false",
|
|
id);
|
|
if (updated == 0) {
|
|
throw new RuntimeException("文档不存在或已被删除");
|
|
}
|
|
|
|
// 查询最新状态
|
|
KnowledgeDocument doc = documentMapper.selectById(id);
|
|
if (doc == null) {
|
|
throw new RuntimeException("文档不存在");
|
|
}
|
|
|
|
// 同步更新 vector_store 中的 enabled metadata
|
|
syncVectorEnabledMetadata(String.valueOf(id), Boolean.TRUE.equals(doc.getEnabled()));
|
|
log.info("文档状态切换: id={}, enabled={}", id, doc.getEnabled());
|
|
return doc;
|
|
}
|
|
|
|
/**
|
|
* P1-2.1: 批量启用/禁用文档
|
|
*/
|
|
public Map<String, Object> batchToggleDocuments(List<Long> ids, boolean enabled) {
|
|
int successCount = 0;
|
|
int failCount = 0;
|
|
for (Long id : ids) {
|
|
try {
|
|
KnowledgeDocument doc = documentMapper.selectById(id);
|
|
if (doc == null) {
|
|
failCount++;
|
|
continue;
|
|
}
|
|
doc.setEnabled(enabled);
|
|
documentMapper.updateById(doc);
|
|
syncVectorEnabledMetadata(String.valueOf(id), enabled);
|
|
successCount++;
|
|
} catch (Exception e) {
|
|
failCount++;
|
|
log.warn("批量切换文档状态失败: id={}", id, e);
|
|
}
|
|
}
|
|
Map<String, Object> result = new HashMap<>();
|
|
result.put("successCount", successCount);
|
|
result.put("failCount", failCount);
|
|
return result;
|
|
}
|
|
|
|
/**
|
|
* 同步更新 vector_store 中指定文档所有分块的 enabled metadata
|
|
* 使用 JdbcTemplate 直接 UPDATE JSON 字段
|
|
* 注意：Spring AI 的 PgVectorStore 自动建表时 metadata 是 json 类型，不是 jsonb
|
|
* 统一写入字符串类型 "true"/"false"，与 searchDocuments 的过滤条件保持一致
|
|
*/
|
|
private void syncVectorEnabledMetadata(String documentId, boolean enabled) {
|
|
// 使用 to_jsonb(?::text) 将字符串转换为 JSONB 字符串值，与 searchDocuments 过滤条件一致
|
|
String sql = "UPDATE vector_store SET metadata = (jsonb_set(metadata::jsonb, '{enabled}', " +
|
|
"to_jsonb(?::text)))::json WHERE metadata->>'documentId' = ?";
|
|
jdbcTemplate.update(sql, enabled ? "true" : "false", documentId);
|
|
}
|
|
|
|
// ==================== 批量移动分类 ====================
|
|
|
|
/**
|
|
* P1-2.2: 批量移动文档到目标分类
|
|
* 仅更新 category_id，不影响向量数据
|
|
*/
|
|
@Transactional(rollbackFor = Exception.class)
|
|
public int batchMoveDocuments(List<Long> ids, Long categoryId) {
|
|
Long targetCategoryId = categoryId != null ? categoryId : 0L;
|
|
int updated = 0;
|
|
for (Long id : ids) {
|
|
KnowledgeDocument doc = documentMapper.selectById(id);
|
|
if (doc != null) {
|
|
doc.setCategoryId(targetCategoryId);
|
|
documentMapper.updateById(doc);
|
|
updated++;
|
|
}
|
|
}
|
|
log.info("批量移动文档分类: ids={}, categoryId={}, 实际更新={}", ids, targetCategoryId, updated);
|
|
return updated;
|
|
}
|
|
|
|
// ==================== 标签管理 ====================
|
|
|
|
/**
|
|
* P1-2.3: 获取标签列表（从所有文档的 tags JSONB 聚合去重，含使用次数）
|
|
*
|
|
* @return [{tag: "标签名", count: 使用次数}]
|
|
*/
|
|
public List<Map<String, Object>> getTagList() {
|
|
String sql = "SELECT tag_value AS tag, COUNT(*) AS count " +
|
|
"FROM knowledge_document, jsonb_array_elements(tags->'tags') AS tag_value " +
|
|
"WHERE is_delete = false AND tags IS NOT NULL AND tags->'tags' IS NOT NULL " +
|
|
"GROUP BY tag_value ORDER BY count DESC, tag_value ASC";
|
|
try {
|
|
return jdbcTemplate.queryForList(sql);
|
|
} catch (Exception e) {
|
|
log.warn("获取标签列表失败", e);
|
|
return List.of();
|
|
}
|
|
}
|
|
|
|
// ==================== 分块管理 ====================
|
|
|
|
/**
|
|
* P1-2.5: 更新单个分块内容（同步更新 vector_store 并重新向量化）
|
|
*/
|
|
@Transactional(rollbackFor = Exception.class)
|
|
public void updateChunk(Long docId, int chunkIndex, String newContent) {
|
|
KnowledgeDocument doc = documentMapper.selectById(docId);
|
|
if (doc == null) {
|
|
throw new RuntimeException("文档不存在");
|
|
}
|
|
|
|
String docIdStr = String.valueOf(docId);
|
|
// 查找对应的向量记录 ID
|
|
String selectSql = "SELECT id::text FROM vector_store " +
|
|
"WHERE (metadata->>'documentId' = ? OR metadata->>'document_id' = ?) " +
|
|
"AND metadata->>'chunkIndex' = ? " +
|
|
"LIMIT 1";
|
|
List<String> vectorIds = jdbcTemplate.queryForList(selectSql, String.class,
|
|
docIdStr, docIdStr, String.valueOf(chunkIndex));
|
|
|
|
if (vectorIds.isEmpty()) {
|
|
throw new RuntimeException("未找到对应的分块向量记录");
|
|
}
|
|
|
|
String vectorId = vectorIds.get(0);
|
|
|
|
// 删除旧向量
|
|
pgVectorVectorStore.delete(List.of(vectorId));
|
|
|
|
// 构建新文档并重新向量化
|
|
Map<String, Object> meta = new HashMap<>();
|
|
meta.put("documentId", docIdStr);
|
|
meta.put("chunkIndex", chunkIndex);
|
|
meta.put("sourceName", doc.getSourceName());
|
|
meta.put("title", doc.getTitle());
|
|
if (doc.getCategoryId() != null && doc.getCategoryId() > 0) {
|
|
meta.put("categoryId", String.valueOf(doc.getCategoryId()));
|
|
}
|
|
if (doc.getTags() != null && doc.getTags().containsKey("tags")) {
|
|
meta.put("tags", doc.getTags().get("tags"));
|
|
}
|
|
// 标记启用状态，与文档当前状态保持一致
|
|
meta.put("enabled", String.valueOf(Boolean.TRUE.equals(doc.getEnabled())));
|
|
|
|
Document newDoc = new Document(vectorId, newContent, meta);
|
|
// 关键词提取
|
|
List<Document> enriched = myKeywordEnricher.enrichDocuments(List.of(newDoc));
|
|
pgVectorVectorStore.add(enriched);
|
|
|
|
log.info("更新分块: docId={}, chunkIndex={}, vectorId={}", docId, chunkIndex, vectorId);
|
|
}
|
|
|
|
/**
|
|
* P1-2.5: 删除单个分块（同步从 vector_store 移除向量）
|
|
*/
|
|
@Transactional(rollbackFor = Exception.class)
|
|
public void deleteChunk(Long docId, int chunkIndex) {
|
|
KnowledgeDocument doc = documentMapper.selectById(docId);
|
|
if (doc == null) {
|
|
throw new RuntimeException("文档不存在");
|
|
}
|
|
|
|
String docIdStr = String.valueOf(docId);
|
|
// 查找对应的向量记录 ID
|
|
String selectSql = "SELECT id::text FROM vector_store " +
|
|
"WHERE (metadata->>'documentId' = ? OR metadata->>'document_id' = ?) " +
|
|
"AND metadata->>'chunkIndex' = ?";
|
|
List<String> vectorIds = jdbcTemplate.queryForList(selectSql, String.class,
|
|
docIdStr, docIdStr, String.valueOf(chunkIndex));
|
|
|
|
if (!vectorIds.isEmpty()) {
|
|
pgVectorVectorStore.delete(vectorIds);
|
|
}
|
|
|
|
// 更新文档的分块计数
|
|
doc.setChunkCount(Math.max(0, (doc.getChunkCount() != null ? doc.getChunkCount() : 1) - vectorIds.size()));
|
|
documentMapper.updateById(doc);
|
|
|
|
log.info("删除分块: docId={}, chunkIndex={}, 删除向量数={}", docId, chunkIndex, vectorIds.size());
|
|
}
|
|
|
|
// ==================== 语义搜索 ====================
|
|
|
|
/**
|
|
* 语义搜索
|
|
*/
|
|
public List<SearchResult> searchDocuments(String query, int topK, double similarityThreshold, Long categoryId) {
|
|
List<Long> categoryIds = categoryId != null ? List.of(categoryId) : Collections.emptyList();
|
|
return searchDocuments(query, topK, similarityThreshold, categoryIds);
|
|
}
|
|
|
|
/**
|
|
* 语义搜索（支持多个知识库分类过滤）
|
|
*/
|
|
public List<SearchResult> searchDocuments(String query, int topK, double similarityThreshold, List<Long> categoryIds) {
|
|
SearchRequest.Builder searchBuilder = SearchRequest.builder()
|
|
.query(query)
|
|
.topK(topK)
|
|
.similarityThreshold(similarityThreshold);
|
|
|
|
FilterExpressionBuilder builder = new FilterExpressionBuilder();
|
|
|
|
// P1-2.1: 始终过滤禁用的文档，只检索启用的
|
|
var enabledFilter = builder.eq("enabled", "true");
|
|
|
|
List<String> categoryIdStrings = normalizeCategoryIds(categoryIds);
|
|
if (!categoryIdStrings.isEmpty()) {
|
|
List<Object> values = categoryIdStrings.stream()
|
|
.map(value -> (Object) value)
|
|
.collect(Collectors.toList());
|
|
var categoryFilter = builder.in("categoryId", values);
|
|
searchBuilder.filterExpression(builder.and(enabledFilter, categoryFilter).build());
|
|
} else {
|
|
searchBuilder.filterExpression(enabledFilter.build());
|
|
}
|
|
|
|
List<Document> results;
|
|
try {
|
|
results = pgVectorVectorStore.similaritySearch(searchBuilder.build());
|
|
} catch (Exception e) {
|
|
if (categoryIdStrings.isEmpty()) {
|
|
throw e;
|
|
}
|
|
log.warn("向量检索分类过滤失败，改用本地 metadata 过滤兜底: categoryIds={}", categoryIdStrings, e);
|
|
results = pgVectorVectorStore.similaritySearch(SearchRequest.builder()
|
|
.query(query)
|
|
.topK(Math.max(topK * 5, 20))
|
|
.similarityThreshold(similarityThreshold)
|
|
.build())
|
|
.stream()
|
|
.filter(doc -> categoryIdStrings.contains(getStringFromMetadata(doc.getMetadata(), "categoryId")))
|
|
.filter(doc -> "true".equals(getStringFromMetadata(doc.getMetadata(), "enabled")))
|
|
.limit(topK)
|
|
.collect(Collectors.toList());
|
|
}
|
|
|
|
List<SearchResult> searchResults = new ArrayList<>();
|
|
for (Document doc : results) {
|
|
Map<String, Object> metadata = doc.getMetadata();
|
|
SearchResult result = SearchResult.builder()
|
|
.id(doc.getId())
|
|
.content(doc.getText())
|
|
.score(metadata.containsKey("distance") ? ((Number) metadata.get("distance")).doubleValue() : null)
|
|
.sourceName(getStringFromMetadata(metadata, "sourceName"))
|
|
.title(getStringFromMetadata(metadata, "title"))
|
|
.chunkIndex(getIntegerFromMetadata(metadata, "chunkIndex"))
|
|
.documentId(getStringFromMetadata(metadata, "documentId"))
|
|
.metadata(metadata)
|
|
.build();
|
|
searchResults.add(result);
|
|
}
|
|
|
|
return searchResults;
|
|
}
|
|
|
|
/**
|
|
* 多模式搜索（P0-001: 支持向量/关键词/混合检索）
|
|
*
|
|
* @param searchMode 检索模式：VECTOR（默认）/ KEYWORD / HYBRID
|
|
*/
|
|
public List<SearchResult> searchDocuments(String query, int topK, double similarityThreshold,
|
|
List<Long> categoryIds, String searchMode) {
|
|
if (searchMode == null || searchMode.isEmpty() || "VECTOR".equalsIgnoreCase(searchMode)) {
|
|
// 默认向量检索，向后兼容
|
|
List<SearchResult> results = searchDocuments(query, topK, similarityThreshold, categoryIds);
|
|
results.forEach(r -> r.setSearchMode("VECTOR"));
|
|
return results;
|
|
}
|
|
|
|
// 委托给 HybridSearchService 处理 KEYWORD 和 HYBRID 模式
|
|
if (hybridSearchService != null) {
|
|
com.wok.supportbot.rag.SearchMode mode;
|
|
try {
|
|
mode = com.wok.supportbot.rag.SearchMode.valueOf(searchMode.toUpperCase());
|
|
} catch (IllegalArgumentException e) {
|
|
mode = com.wok.supportbot.rag.SearchMode.VECTOR;
|
|
}
|
|
return hybridSearchService.search(query, mode, topK, similarityThreshold, categoryIds);
|
|
}
|
|
|
|
// HybridSearchService 不可用时降级到向量检索
|
|
log.warn("HybridSearchService 不可用，降级为向量检索");
|
|
return searchDocuments(query, topK, similarityThreshold, categoryIds);
|
|
}
|
|
|
|
@Autowired(required = false)
|
|
private com.wok.supportbot.rag.HybridSearchService hybridSearchService;
|
|
|
|
private List<String> normalizeCategoryIds(List<Long> categoryIds) {
|
|
if (categoryIds == null || categoryIds.isEmpty()) {
|
|
return Collections.emptyList();
|
|
}
|
|
return categoryIds.stream()
|
|
.filter(Objects::nonNull)
|
|
.filter(id -> id > 0)
|
|
.map(String::valueOf)
|
|
.distinct()
|
|
.collect(Collectors.toList());
|
|
}
|
|
|
|
// ==================== 统计 ====================
|
|
|
|
/**
|
|
* 获取知识库统计信息
|
|
*/
|
|
public Map<String, Object> getStats() {
|
|
// 文档统计
|
|
Long totalDocuments = documentMapper.selectCount(null);
|
|
|
|
// 按文件类型统计
|
|
String typeSql = "SELECT file_type, COUNT(*) as count FROM knowledge_document WHERE is_delete = false GROUP BY file_type";
|
|
List<Map<String, Object>> typeStats = jdbcTemplate.queryForList(typeSql);
|
|
Map<String, Long> byFileType = typeStats.stream()
|
|
.collect(Collectors.toMap(
|
|
r -> (String) r.get("file_type"),
|
|
r -> ((Number) r.get("count")).longValue()
|
|
));
|
|
|
|
// 按分类统计
|
|
String catSql = "SELECT c.name, COUNT(d.id) as count FROM knowledge_document d " +
|
|
"LEFT JOIN knowledge_category c ON d.category_id = c.id " +
|
|
"WHERE d.is_delete = false GROUP BY c.name";
|
|
List<Map<String, Object>> catStats;
|
|
try {
|
|
catStats = jdbcTemplate.queryForList(catSql);
|
|
} catch (Exception e) {
|
|
catStats = new ArrayList<>();
|
|
}
|
|
|
|
// 向量总数
|
|
String vectorSql = "SELECT COUNT(*) FROM vector_store";
|
|
Long totalVectors;
|
|
try {
|
|
totalVectors = jdbcTemplate.queryForObject(vectorSql, Long.class);
|
|
} catch (Exception e) {
|
|
totalVectors = 0L;
|
|
}
|
|
|
|
// 最近上传时间
|
|
String lastUploadSql = "SELECT MAX(create_time) FROM knowledge_document WHERE is_delete = false";
|
|
Date lastUploadTime = jdbcTemplate.queryForObject(lastUploadSql, Date.class);
|
|
|
|
Map<String, Object> stats = new LinkedHashMap<>();
|
|
stats.put("totalDocuments", totalDocuments);
|
|
stats.put("totalVectors", totalVectors);
|
|
stats.put("lastUploadTime", lastUploadTime);
|
|
stats.put("byFileType", byFileType);
|
|
stats.put("byCategory", catStats);
|
|
|
|
return stats;
|
|
}
|
|
|
|
// ==================== 分类管理 ====================
|
|
|
|
/**
|
|
* 获取分类树
|
|
*/
|
|
public List<CategoryNode> getCategoryTree() {
|
|
List<KnowledgeCategory> categories = categoryMapper.selectList(
|
|
new QueryWrapper<KnowledgeCategory>().orderByAsc("sort_order"));
|
|
|
|
Map<Long, CategoryNode> nodeMap = new LinkedHashMap<>();
|
|
List<CategoryNode> rootNodes = new ArrayList<>();
|
|
|
|
for (KnowledgeCategory cat : categories) {
|
|
CategoryNode node = CategoryNode.builder()
|
|
.id(cat.getId())
|
|
.name(cat.getName())
|
|
.description(cat.getDescription())
|
|
.parentId(cat.getParentId())
|
|
.sortOrder(cat.getSortOrder())
|
|
.documentCount(cat.getDocumentCount())
|
|
.children(new ArrayList<>())
|
|
.build();
|
|
nodeMap.put(cat.getId(), node);
|
|
}
|
|
|
|
for (CategoryNode node : nodeMap.values()) {
|
|
if (node.getParentId() == null || node.getParentId() == 0) {
|
|
rootNodes.add(node);
|
|
} else {
|
|
CategoryNode parent = nodeMap.get(node.getParentId());
|
|
if (parent != null) {
|
|
parent.getChildren().add(node);
|
|
} else {
|
|
rootNodes.add(node);
|
|
}
|
|
}
|
|
}
|
|
|
|
return rootNodes;
|
|
}
|
|
|
|
/**
|
|
* 获取分类列表
|
|
*/
|
|
public List<KnowledgeCategory> listCategories() {
|
|
return categoryMapper.selectList(
|
|
new QueryWrapper<KnowledgeCategory>().orderByAsc("sort_order"));
|
|
}
|
|
|
|
/**
|
|
* 创建分类
|
|
*/
|
|
public KnowledgeCategory createCategory(String name, String description, Long parentId, Integer sortOrder) {
|
|
KnowledgeCategory category = KnowledgeCategory.builder()
|
|
.name(name)
|
|
.description(description)
|
|
.parentId(parentId != null ? parentId : 0L)
|
|
.sortOrder(sortOrder != null ? sortOrder : 0)
|
|
.documentCount(0)
|
|
.build();
|
|
categoryMapper.insert(category);
|
|
return category;
|
|
}
|
|
|
|
/**
|
|
* 更新分类
|
|
*/
|
|
public void updateCategory(Long id, String name, String description, Integer sortOrder) {
|
|
KnowledgeCategory category = categoryMapper.selectById(id);
|
|
if (category == null) {
|
|
throw new RuntimeException("分类不存在");
|
|
}
|
|
if (name != null && !name.isEmpty()) {
|
|
category.setName(name);
|
|
}
|
|
if (description != null) {
|
|
category.setDescription(description);
|
|
}
|
|
if (sortOrder != null) {
|
|
category.setSortOrder(sortOrder);
|
|
}
|
|
categoryMapper.updateById(category);
|
|
}
|
|
|
|
/**
|
|
* 删除分类（不删除文档，仅清空关联）
|
|
*/
|
|
@Transactional(rollbackFor = Exception.class)
|
|
public void deleteCategory(Long id) {
|
|
// 将关联的文档 category_id 设为 0
|
|
KnowledgeDocument updateDoc = new KnowledgeDocument();
|
|
updateDoc.setCategoryId(0L);
|
|
documentMapper.update(updateDoc, new QueryWrapper<KnowledgeDocument>().eq("category_id", id));
|
|
|
|
// 逻辑删除分类
|
|
categoryMapper.deleteById(id);
|
|
}
|
|
|
|
// ==================== 内部方法 ====================
|
|
|
|
/**
|
|
* 计算文本内容的 SHA-256 哈希值
|
|
*/
|
|
private String computeContentHash(String content) {
|
|
if (content == null || content.isEmpty()) {
|
|
return null;
|
|
}
|
|
try {
|
|
MessageDigest digest = MessageDigest.getInstance("SHA-256");
|
|
byte[] hashBytes = digest.digest(content.getBytes(StandardCharsets.UTF_8));
|
|
StringBuilder sb = new StringBuilder();
|
|
for (byte b : hashBytes) {
|
|
sb.append(String.format("%02x", b));
|
|
}
|
|
return sb.toString();
|
|
} catch (NoSuchAlgorithmException e) {
|
|
log.error("SHA-256 算法不可用", e);
|
|
return null;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* 检查内容是否重复
|
|
* @param contentHash 内容哈希值
|
|
* @return 重复文档的标题，如果不存在重复则返回 null
|
|
*/
|
|
private String checkContentDuplicate(String contentHash) {
|
|
if (contentHash == null) {
|
|
return null;
|
|
}
|
|
QueryWrapper<KnowledgeDocument> wrapper = new QueryWrapper<>();
|
|
wrapper.eq("content_hash", contentHash);
|
|
wrapper.select("title");
|
|
KnowledgeDocument existing = documentMapper.selectOne(wrapper);
|
|
return existing != null ? existing.getTitle() : null;
|
|
}
|
|
|
|
/**
|
|
* 根据文档ID删除 vector_store 中关联的所有向量
|
|
*/
|
|
private int deleteVectorsByDocumentId(String documentId) {
|
|
String sql = "SELECT id::text FROM vector_store WHERE metadata->>'documentId' = ?";
|
|
List<String> ids = jdbcTemplate.queryForList(sql, String.class, documentId);
|
|
|
|
if (!ids.isEmpty()) {
|
|
pgVectorVectorStore.delete(ids);
|
|
log.debug("删除向量: documentId={}, count={}", documentId, ids.size());
|
|
}
|
|
return ids.size();
|
|
}
|
|
|
|
/**
|
|
* 获取文件扩展名
|
|
*/
|
|
private String getFileExtension(String filename) {
|
|
if (filename == null || !filename.contains(".")) {
|
|
return "unknown";
|
|
}
|
|
return filename.substring(filename.lastIndexOf(".") + 1).toLowerCase();
|
|
}
|
|
|
|
/**
|
|
* 从 metadata 中安全获取字符串值
|
|
*/
|
|
private String getStringFromMetadata(Map<String, Object> metadata, String key) {
|
|
Object value = metadata.get(key);
|
|
return value != null ? value.toString() : null;
|
|
}
|
|
|
|
/**
|
|
* 从 metadata 中安全获取整数值
|
|
*/
|
|
private Integer getIntegerFromMetadata(Map<String, Object> metadata, String key) {
|
|
Object value = metadata.get(key);
|
|
if (value == null) return null;
|
|
if (value instanceof Number) {
|
|
return ((Number) value).intValue();
|
|
}
|
|
try {
|
|
return Integer.parseInt(value.toString());
|
|
} catch (NumberFormatException e) {
|
|
return null;
|
|
}
|
|
}
|
|
}
|