|
|
@ -73,6 +73,9 @@ public class DocumentService { |
|
|
@Autowired |
|
|
@Autowired |
|
|
private DocumentProcessingService documentProcessingService; |
|
|
private DocumentProcessingService documentProcessingService; |
|
|
|
|
|
|
|
|
|
|
|
@Autowired |
|
|
|
|
|
private com.wok.supportbot.config.FileStorageConfig fileStorageConfig; |
|
|
|
|
|
|
|
|
// ==================== 文档上传 ==================== |
|
|
// ==================== 文档上传 ==================== |
|
|
|
|
|
|
|
|
/** |
|
|
/** |
|
|
@ -90,13 +93,15 @@ public class DocumentService { |
|
|
* @param tags 标签列表 |
|
|
* @param tags 标签列表 |
|
|
* @param chunkSize 分块大小(可选,覆盖全局配置) |
|
|
* @param chunkSize 分块大小(可选,覆盖全局配置) |
|
|
* @param overlap 重叠大小(可选,覆盖全局配置) |
|
|
* @param overlap 重叠大小(可选,覆盖全局配置) |
|
|
|
|
|
* @param filePath 原始文件存储路径(可选,null表示无原始文件) |
|
|
* @return 创建完成的文档记录(status=PROCESSING) |
|
|
* @return 创建完成的文档记录(status=PROCESSING) |
|
|
*/ |
|
|
*/ |
|
|
@Transactional(rollbackFor = Exception.class) |
|
|
@Transactional(rollbackFor = Exception.class) |
|
|
public KnowledgeDocument uploadDocument(List<Document> documents, String title, String sourceName, |
|
|
public KnowledgeDocument uploadDocument(List<Document> documents, String title, String sourceName, |
|
|
String fileType, Long fileSize, String content, |
|
|
String fileType, Long fileSize, String content, |
|
|
Long categoryId, List<String> tags, |
|
|
Long categoryId, List<String> tags, |
|
|
Integer chunkSize, Integer overlap) { |
|
|
|
|
|
|
|
|
Integer chunkSize, Integer overlap, |
|
|
|
|
|
String filePath) { |
|
|
// 0. 内容去重检查 |
|
|
// 0. 内容去重检查 |
|
|
String contentHash = computeContentHash(content); |
|
|
String contentHash = computeContentHash(content); |
|
|
if (contentHash != null) { |
|
|
if (contentHash != null) { |
|
|
@ -117,6 +122,7 @@ public class DocumentService { |
|
|
.sourceName(sourceName) |
|
|
.sourceName(sourceName) |
|
|
.fileType(fileType) |
|
|
.fileType(fileType) |
|
|
.fileSize(fileSize != null ? fileSize : 0L) |
|
|
.fileSize(fileSize != null ? fileSize : 0L) |
|
|
|
|
|
.filePath(filePath) |
|
|
.content(content != null && content.length() > 2000 ? content.substring(0, 2000) : content) |
|
|
.content(content != null && content.length() > 2000 ? content.substring(0, 2000) : content) |
|
|
.categoryId(categoryId != null ? categoryId : 0L) |
|
|
.categoryId(categoryId != null ? categoryId : 0L) |
|
|
.tags(tags != null ? Map.of("tags", tags) : null) |
|
|
.tags(tags != null ? Map.of("tags", tags) : null) |
|
|
@ -138,12 +144,19 @@ public class DocumentService { |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
/** |
|
|
/** |
|
|
* 解析文件并上传 |
|
|
|
|
|
|
|
|
* 解析文件并上传(同时保存原始文件到本地) |
|
|
*/ |
|
|
*/ |
|
|
public KnowledgeDocument uploadFile(MultipartFile file, String title, Long categoryId, List<String> tags, |
|
|
public KnowledgeDocument uploadFile(MultipartFile file, String title, Long categoryId, List<String> tags, |
|
|
Integer chunkSize, Integer overlap) { |
|
|
Integer chunkSize, Integer overlap) { |
|
|
List<Document> documents = tikaDocumentReader.read(file); |
|
|
|
|
|
|
|
|
// 1. 先保存原始文件到本地磁盘(必须在解析之前,因为 transferTo 只能调用一次) |
|
|
|
|
|
String relativePath = saveFileToLocal(file); |
|
|
|
|
|
java.io.File savedFile = fileStorageConfig.getFilePath(relativePath).toFile(); |
|
|
|
|
|
|
|
|
|
|
|
// 2. 使用保存的文件进行解析(避免 MultipartFile 的临时文件问题) |
|
|
|
|
|
List<Document> documents = tikaDocumentReader.readFromFile(savedFile); |
|
|
String fileType = getFileExtension(file.getOriginalFilename()); |
|
|
String fileType = getFileExtension(file.getOriginalFilename()); |
|
|
|
|
|
|
|
|
|
|
|
// 3. 创建文档记录并保存文件路径 |
|
|
return uploadDocument(documents, |
|
|
return uploadDocument(documents, |
|
|
title != null ? title : file.getOriginalFilename(), |
|
|
title != null ? title : file.getOriginalFilename(), |
|
|
file.getOriginalFilename(), |
|
|
file.getOriginalFilename(), |
|
|
@ -151,17 +164,45 @@ public class DocumentService { |
|
|
file.getSize(), |
|
|
file.getSize(), |
|
|
documents.get(0).getText(), |
|
|
documents.get(0).getText(), |
|
|
categoryId, |
|
|
categoryId, |
|
|
tags, chunkSize, overlap); |
|
|
|
|
|
|
|
|
tags, chunkSize, overlap, |
|
|
|
|
|
relativePath); |
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
/** |
|
|
|
|
|
* 保存上传的文件到本地磁盘 |
|
|
|
|
|
* |
|
|
|
|
|
* @param file 上传的文件 |
|
|
|
|
|
* @return 相对路径 |
|
|
|
|
|
*/ |
|
|
|
|
|
private String saveFileToLocal(MultipartFile file) { |
|
|
|
|
|
try { |
|
|
|
|
|
String relativePath = fileStorageConfig.generateRelativePath(file.getOriginalFilename()); |
|
|
|
|
|
java.nio.file.Path targetPath = fileStorageConfig.getFilePath(relativePath); |
|
|
|
|
|
|
|
|
|
|
|
// 确保父目录存在 |
|
|
|
|
|
java.io.File parentDir = targetPath.getParent().toFile(); |
|
|
|
|
|
if (!parentDir.exists()) { |
|
|
|
|
|
parentDir.mkdirs(); |
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
// 使用 transferTo 保存文件(这是唯一一次调用) |
|
|
|
|
|
file.transferTo(targetPath.toFile()); |
|
|
|
|
|
log.info("文件已保存到本地: {}", targetPath); |
|
|
|
|
|
return relativePath; |
|
|
|
|
|
} catch (java.io.IOException e) { |
|
|
|
|
|
log.error("保存文件到本地失败", e); |
|
|
|
|
|
throw new RuntimeException("保存文件失败: " + e.getMessage(), e); |
|
|
|
|
|
} |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
/** |
|
|
/** |
|
|
* 解析字符串并上传 |
|
|
|
|
|
|
|
|
* 解析字符串并上传(无原始文件) |
|
|
*/ |
|
|
*/ |
|
|
public KnowledgeDocument uploadString(String content, String title, Long categoryId, List<String> tags, |
|
|
public KnowledgeDocument uploadString(String content, String title, Long categoryId, List<String> tags, |
|
|
Integer chunkSize, Integer overlap) { |
|
|
Integer chunkSize, Integer overlap) { |
|
|
List<Document> documents = simpleStringDocumentReader.read(content); |
|
|
List<Document> documents = simpleStringDocumentReader.read(content); |
|
|
return uploadDocument(documents, title, title, "txt", |
|
|
return uploadDocument(documents, title, title, "txt", |
|
|
(long) content.length(), content, categoryId, tags, chunkSize, overlap); |
|
|
|
|
|
|
|
|
(long) content.length(), content, categoryId, tags, chunkSize, overlap, null); |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
/** |
|
|
/** |
|
|
@ -169,8 +210,14 @@ public class DocumentService { |
|
|
*/ |
|
|
*/ |
|
|
public KnowledgeDocument uploadMarkdown(MultipartFile file, String title, Long categoryId, List<String> tags, |
|
|
public KnowledgeDocument uploadMarkdown(MultipartFile file, String title, Long categoryId, List<String> tags, |
|
|
Integer chunkSize, Integer overlap) { |
|
|
Integer chunkSize, Integer overlap) { |
|
|
List<Document> documents = markdownDocumentLoader.loadMarkdownFromFile(file); |
|
|
|
|
|
|
|
|
// 1. 先保存原始文件 |
|
|
|
|
|
String relativePath = saveFileToLocal(file); |
|
|
|
|
|
java.io.File savedFile = fileStorageConfig.getFilePath(relativePath).toFile(); |
|
|
|
|
|
|
|
|
|
|
|
// 2. 使用保存的文件进行解析 |
|
|
|
|
|
List<Document> documents = markdownDocumentLoader.loadMarkdownFromFile(savedFile); |
|
|
String content = documents.stream().map(Document::getText).collect(Collectors.joining("\n")); |
|
|
String content = documents.stream().map(Document::getText).collect(Collectors.joining("\n")); |
|
|
|
|
|
|
|
|
return uploadDocument(documents, |
|
|
return uploadDocument(documents, |
|
|
title != null ? title : file.getOriginalFilename(), |
|
|
title != null ? title : file.getOriginalFilename(), |
|
|
file.getOriginalFilename(), |
|
|
file.getOriginalFilename(), |
|
|
@ -178,7 +225,8 @@ public class DocumentService { |
|
|
file.getSize(), |
|
|
file.getSize(), |
|
|
content, |
|
|
content, |
|
|
categoryId, |
|
|
categoryId, |
|
|
tags, chunkSize, overlap); |
|
|
|
|
|
|
|
|
tags, chunkSize, overlap, |
|
|
|
|
|
relativePath); |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
/** |
|
|
/** |
|
|
@ -186,8 +234,14 @@ public class DocumentService { |
|
|
*/ |
|
|
*/ |
|
|
public KnowledgeDocument uploadJsonBasic(MultipartFile file, String title, Long categoryId, List<String> tags, |
|
|
public KnowledgeDocument uploadJsonBasic(MultipartFile file, String title, Long categoryId, List<String> tags, |
|
|
Integer chunkSize, Integer overlap) { |
|
|
Integer chunkSize, Integer overlap) { |
|
|
List<Document> documents = jsonDocumentLoader.loadBasicJson(file); |
|
|
|
|
|
|
|
|
// 1. 先保存原始文件 |
|
|
|
|
|
String relativePath = saveFileToLocal(file); |
|
|
|
|
|
java.io.File savedFile = fileStorageConfig.getFilePath(relativePath).toFile(); |
|
|
|
|
|
|
|
|
|
|
|
// 2. 使用保存的文件进行解析 |
|
|
|
|
|
List<Document> documents = jsonDocumentLoader.loadBasicJsonFromFile(savedFile); |
|
|
String content = documents.stream().map(Document::getText).collect(Collectors.joining("\n")); |
|
|
String content = documents.stream().map(Document::getText).collect(Collectors.joining("\n")); |
|
|
|
|
|
|
|
|
return uploadDocument(documents, |
|
|
return uploadDocument(documents, |
|
|
title != null ? title : file.getOriginalFilename(), |
|
|
title != null ? title : file.getOriginalFilename(), |
|
|
file.getOriginalFilename(), |
|
|
file.getOriginalFilename(), |
|
|
@ -195,7 +249,8 @@ public class DocumentService { |
|
|
file.getSize(), |
|
|
file.getSize(), |
|
|
content, |
|
|
content, |
|
|
categoryId, |
|
|
categoryId, |
|
|
tags, chunkSize, overlap); |
|
|
|
|
|
|
|
|
tags, chunkSize, overlap, |
|
|
|
|
|
relativePath); |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
/** |
|
|
/** |
|
|
@ -204,8 +259,14 @@ public class DocumentService { |
|
|
public KnowledgeDocument uploadJsonFields(MultipartFile file, List<String> fields, String title, |
|
|
public KnowledgeDocument uploadJsonFields(MultipartFile file, List<String> fields, String title, |
|
|
Long categoryId, List<String> tags, |
|
|
Long categoryId, List<String> tags, |
|
|
Integer chunkSize, Integer overlap) { |
|
|
Integer chunkSize, Integer overlap) { |
|
|
List<Document> documents = jsonDocumentLoader.loadJsonByFields(file, fields.toArray(new String[0])); |
|
|
|
|
|
|
|
|
// 1. 先保存原始文件 |
|
|
|
|
|
String relativePath = saveFileToLocal(file); |
|
|
|
|
|
java.io.File savedFile = fileStorageConfig.getFilePath(relativePath).toFile(); |
|
|
|
|
|
|
|
|
|
|
|
// 2. 使用保存的文件进行解析 |
|
|
|
|
|
List<Document> documents = jsonDocumentLoader.loadJsonByFieldsFromFile(savedFile, fields.toArray(new String[0])); |
|
|
String content = documents.stream().map(Document::getText).collect(Collectors.joining("\n")); |
|
|
String content = documents.stream().map(Document::getText).collect(Collectors.joining("\n")); |
|
|
|
|
|
|
|
|
return uploadDocument(documents, |
|
|
return uploadDocument(documents, |
|
|
title != null ? title : file.getOriginalFilename(), |
|
|
title != null ? title : file.getOriginalFilename(), |
|
|
file.getOriginalFilename(), |
|
|
file.getOriginalFilename(), |
|
|
@ -213,7 +274,8 @@ public class DocumentService { |
|
|
file.getSize(), |
|
|
file.getSize(), |
|
|
content, |
|
|
content, |
|
|
categoryId, |
|
|
categoryId, |
|
|
tags, chunkSize, overlap); |
|
|
|
|
|
|
|
|
tags, chunkSize, overlap, |
|
|
|
|
|
relativePath); |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
/** |
|
|
/** |
|
|
@ -222,8 +284,14 @@ public class DocumentService { |
|
|
public KnowledgeDocument uploadJsonPointer(MultipartFile file, String pointer, String title, |
|
|
public KnowledgeDocument uploadJsonPointer(MultipartFile file, String pointer, String title, |
|
|
Long categoryId, List<String> tags, |
|
|
Long categoryId, List<String> tags, |
|
|
Integer chunkSize, Integer overlap) { |
|
|
Integer chunkSize, Integer overlap) { |
|
|
List<Document> documents = jsonDocumentLoader.loadJsonByPointer(file, pointer); |
|
|
|
|
|
|
|
|
// 1. 先保存原始文件 |
|
|
|
|
|
String relativePath = saveFileToLocal(file); |
|
|
|
|
|
java.io.File savedFile = fileStorageConfig.getFilePath(relativePath).toFile(); |
|
|
|
|
|
|
|
|
|
|
|
// 2. 使用保存的文件进行解析 |
|
|
|
|
|
List<Document> documents = jsonDocumentLoader.loadJsonByPointerFromFile(savedFile, pointer); |
|
|
String content = documents.stream().map(Document::getText).collect(Collectors.joining("\n")); |
|
|
String content = documents.stream().map(Document::getText).collect(Collectors.joining("\n")); |
|
|
|
|
|
|
|
|
return uploadDocument(documents, |
|
|
return uploadDocument(documents, |
|
|
title != null ? title : file.getOriginalFilename(), |
|
|
title != null ? title : file.getOriginalFilename(), |
|
|
file.getOriginalFilename(), |
|
|
file.getOriginalFilename(), |
|
|
@ -231,7 +299,33 @@ public class DocumentService { |
|
|
file.getSize(), |
|
|
file.getSize(), |
|
|
content, |
|
|
content, |
|
|
categoryId, |
|
|
categoryId, |
|
|
tags, chunkSize, overlap); |
|
|
|
|
|
|
|
|
tags, chunkSize, overlap, |
|
|
|
|
|
relativePath); |
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
// ==================== 文件下载 ==================== |
|
|
|
|
|
|
|
|
|
|
|
/** |
|
|
|
|
|
* 获取文档的原始文件(用于下载) |
|
|
|
|
|
* |
|
|
|
|
|
* @param documentId 文档ID |
|
|
|
|
|
* @return 文件对象和原始文件名 |
|
|
|
|
|
*/ |
|
|
|
|
|
public java.util.Map.Entry<java.io.File, String> getDocumentFile(Long documentId) { |
|
|
|
|
|
KnowledgeDocument doc = documentMapper.selectById(documentId); |
|
|
|
|
|
if (doc == null || doc.isDelete()) { |
|
|
|
|
|
throw new RuntimeException("文档不存在"); |
|
|
|
|
|
} |
|
|
|
|
|
if (doc.getFilePath() == null || doc.getFilePath().isEmpty()) { |
|
|
|
|
|
throw new RuntimeException("该文档无原始文件(纯文本上传)"); |
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
java.io.File file = fileStorageConfig.getFilePath(doc.getFilePath()).toFile(); |
|
|
|
|
|
if (!file.exists()) { |
|
|
|
|
|
throw new RuntimeException("文件不存在或已被删除"); |
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
return java.util.Map.entry(file, doc.getSourceName()); |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
// ==================== 文档管理 ==================== |
|
|
// ==================== 文档管理 ==================== |
|
|
|