Browse Source
refactor: 自造轮子改用 Spring AI 标准组件、清理死代码并修复既有缺陷
refactor: 自造轮子改用 Spring AI 标准组件、清理死代码并修复既有缺陷
组件替换（改用 1.1.x 标准组件）
- 文档提取：删除自写 TikaDocumentReader（内部直接 new org.apache.tika.Tika()），
改用官方 org.springframework.ai.reader.tika.TikaDocumentReader。
pom 早已引入 spring-ai-tika-document-reader 却从未使用其类，属典型「引了标准依赖却手写实现」
- 意图识别：IntentRouter 的手写正则解析（含去 BOM/零宽字符/全角空格等约 60 行防御逻辑）
改用标准 BeanOutputConverter<IntentResult>，保留原有降级语义（空输入/解析失败 → RAG）
- 推荐问题：SuggestionGenerator 的哨兵字符串 + 代码块剥离 + 按行降级解析改用
ChatClient.entity(ParameterizedTypeReference<List<String>>)，整体删除 SuggestionResponseParser
- 分块：删除 MyTokenTextSplitter 薄壳，新增 OverlapTokenTextSplitter
分块器 overlap 缺陷修复（本次验证中发现并修复）
- 旧实现把 knowledge.chunk.overlap 传进了 TokenTextSplitter 第 2 个形参 minChunkSizeChars 位，
overlap 从未生效（Spring AI 的 TokenTextSplitter 根本没有 overlap 形参）
- 新实现继承标准 TextSplitter，复刻 TokenTextSplitter 全部切分语义，仅把前进步长改为
chunkSize - overlap，使重叠真正生效
- 验证中发现：标点截断会缩短本块消耗的 token 数，与 overlap 叠加后可把前进步长压到
1 个 token，分块数膨胀 10 倍（实测 349 块 vs 修复后 44 块）。故仅当截断后仍能前进
至少 (chunkSize - overlap) / 2 个 token 时才采用该截断，否则宁可切断句子
- overlap=0 时与标准 TokenTextSplitter 逐块一致（已对比验证）
- ⚠️ 存量文档需重新分块 + 重新向量化（POST /document/batch/reprocess），否则新旧向量口径混杂
其他既有缺陷修复
- RerankerService：原用 new RestTemplate() 且无任何超时，慢 provider 会把检索线程拖到
TCP 超时；改用 RestClient + JdkClientHttpRequestFactory 显式设置 connect/read 超时（各 3s）
- McpServerConfigController：MCP Server 增删改/启停/全量刷新后未清 AssistantApp 的
ChatClient 缓存，导致继续使用旧工具集；现补调 clearCache()
死代码清理（均已 grep 确认零引用）
- 删除 FileBasedChatMemory、ReReadingAdvisor（零装配且 before() 逻辑为 no-op）、
SseEventBuilder（零引用）、SuggestionResponseParser
- McpToolCallback 删除 EVENTS ThreadLocal 与 drainEvents()/resetEvents()（零调用），
保留在用的 MCP_EVENTS_KEY/MCP_ROUNDS_KEY ToolContext 机制
文档
- 同步更新 CLAUDE.md / README.md / DEPLOY.md 的版本号、组件说明与架构描述
- 修正 CLAUDE.md 中与代码不符的既有描述：主启动类并未排除 PgVectorStoreAutoConfiguration
（项目用的是非 starter 坐标，classpath 上本就没有该自动配置）；分类过滤实现已完成，
原「Spring AI filter 支持有限」的 TODO 已不成立
Spring-AI-1.1.2
17 changed files with 436 additions and 715 deletions
-
33CLAUDE.md
-
2DEPLOY.md
-
31README.md
-
43src/main/java/com/wok/supportbot/advisor/ReReadingAdvisor.java
-
5src/main/java/com/wok/supportbot/app/ChatResult.java
-
31src/main/java/com/wok/supportbot/app/SuggestionGenerator.java
-
294src/main/java/com/wok/supportbot/app/SuggestionResponseParser.java
-
14src/main/java/com/wok/supportbot/controller/McpServerConfigController.java
-
73src/main/java/com/wok/supportbot/document/extract/TikaDocumentReader.java
-
67src/main/java/com/wok/supportbot/document/transform/MyTokenTextSplitter.java
-
252src/main/java/com/wok/supportbot/document/transform/OverlapTokenTextSplitter.java
-
36src/main/java/com/wok/supportbot/mcp/McpToolCallback.java
-
73src/main/java/com/wok/supportbot/mcp/SseEventBuilder.java
-
50src/main/java/com/wok/supportbot/rag/RerankerService.java
-
34src/main/java/com/wok/supportbot/service/DocumentProcessingService.java
-
13src/main/java/com/wok/supportbot/service/DocumentService.java
-
100src/main/java/com/wok/supportbot/service/IntentRouter.java
@ -1,43 +0,0 @@ |
|||
package com.wok.supportbot.advisor; |
|||
|
|||
import org.springframework.ai.chat.client.ChatClientRequest; |
|||
import org.springframework.ai.chat.client.ChatClientResponse; |
|||
import org.springframework.ai.chat.client.advisor.api.AdvisorChain; |
|||
import org.springframework.ai.chat.client.advisor.api.BaseAdvisor; |
|||
|
|||
import java.util.HashMap; |
|||
import java.util.Map; |
|||
|
|||
/** |
|||
* 自定义 Re2 Advisor（适配 Spring AI 1.0.1 新 Advisor API） |
|||
* 可提高大型语言模型的推理能力 |
|||
*/ |
|||
public class ReReadingAdvisor implements BaseAdvisor { |
|||
|
|||
@Override |
|||
public String getName() { |
|||
return this.getClass().getSimpleName(); |
|||
} |
|||
|
|||
@Override |
|||
public int getOrder() { |
|||
return 0; |
|||
} |
|||
|
|||
@Override |
|||
public ChatClientRequest before(ChatClientRequest request, AdvisorChain chain) { |
|||
// Re2 策略：将用户问题重复一次以增强推理 |
|||
// 通过 context 传递原始查询，在 prompt 中追加重复指令 |
|||
Map<String, Object> newContext = new HashMap<>(request.context()); |
|||
newContext.put("re2_enabled", true); |
|||
return ChatClientRequest.builder() |
|||
.prompt(request.prompt()) |
|||
.context(newContext) |
|||
.build(); |
|||
} |
|||
|
|||
@Override |
|||
public ChatClientResponse after(ChatClientResponse response, AdvisorChain chain) { |
|||
return response; |
|||
} |
|||
} |
|||
@ -1,294 +0,0 @@ |
|||
package com.wok.supportbot.app; |
|||
|
|||
import com.fasterxml.jackson.core.type.TypeReference; |
|||
import com.fasterxml.jackson.databind.ObjectMapper; |
|||
import com.wok.supportbot.cache.SuggestionCache; |
|||
import lombok.extern.slf4j.Slf4j; |
|||
import reactor.core.publisher.Flux; |
|||
|
|||
import java.util.ArrayList; |
|||
import java.util.Collections; |
|||
import java.util.List; |
|||
import java.util.stream.Collectors; |
|||
|
|||
/** |
|||
* 建议问题解析器：从 LLM 原始输出中分离 answer 和 suggestions。 |
|||
* <p> |
|||
* 哨兵标记为 <code>___SUGGESTIONS___</code>，之后为 JSON 字符串数组。 |
|||
* 支持同步路径（直接分割完整文本）和流式路径（滑动窗口检测哨兵）。 |
|||
*/ |
|||
@Slf4j |
|||
public final class SuggestionResponseParser { |
|||
|
|||
/** 主哨兵：严格匹配提示词要求的下划线分隔标记 */ |
|||
private static final String SENTINEL = "___SUGGESTIONS___"; |
|||
|
|||
/** 兼容哨兵：模型偶尔不按指令输出时的大写变形 */ |
|||
private static final String LOOSE_SENTINEL = "SUGGESTIONS"; |
|||
|
|||
private static final ObjectMapper objectMapper = new ObjectMapper(); |
|||
|
|||
private SuggestionResponseParser() { |
|||
} |
|||
|
|||
/** |
|||
* 查找最佳哨兵位置：优先严格哨兵，其次兼容哨兵。 |
|||
* <p> |
|||
* 兼容哨兵必须后面紧跟 JSON 数组特征（'[' 或空白 + '['）， |
|||
* 避免正文中出现 "SUGGESTIONS" 普通单词时误触发。 |
|||
* |
|||
* @param text 要搜索的文本 |
|||
* @return 哨兵起始位置，未找到返回 -1 |
|||
*/ |
|||
private static int findSentinelIndex(String text) { |
|||
int strictIdx = text.lastIndexOf(SENTINEL); |
|||
if (strictIdx >= 0) { |
|||
return strictIdx; |
|||
} |
|||
|
|||
// 兼容模式：从后往前找大写 SUGGESTIONS，且后续需连接 JSON 数组 |
|||
int idx = text.lastIndexOf(LOOSE_SENTINEL); |
|||
while (idx >= 0) { |
|||
int after = idx + LOOSE_SENTINEL.length(); |
|||
if (after < text.length()) { |
|||
char c = text.charAt(after); |
|||
// 允许 SUGGESTIONS[...]、SUGGESTIONS [...]、SUGGESTIONS:\n[...] |
|||
if (c == '[' || Character.isWhitespace(c) || c == ':' || c == '-') { |
|||
return idx; |
|||
} |
|||
} |
|||
// 继续向前查找更早的兼容哨兵 |
|||
idx = text.lastIndexOf(LOOSE_SENTINEL, idx - 1); |
|||
} |
|||
return -1; |
|||
} |
|||
|
|||
/** |
|||
* 同步路径：按最后一次出现哨兵的位置分割完整文本。 |
|||
* |
|||
* @param rawText LLM 原始输出 |
|||
* @return 解析结果（answer + suggestions） |
|||
*/ |
|||
public static ParsedResponse parse(String rawText) { |
|||
if (rawText == null || rawText.isEmpty()) { |
|||
return new ParsedResponse(rawText != null ? rawText : "", Collections.emptyList()); |
|||
} |
|||
|
|||
int lastIdx = findSentinelIndex(rawText); |
|||
if (lastIdx < 0) { |
|||
// 无哨兵标记，整段作为 answer |
|||
return new ParsedResponse(rawText, Collections.emptyList()); |
|||
} |
|||
|
|||
String answer = rawText.substring(0, lastIdx).trim(); |
|||
// 按实际匹配的哨兵长度截取后续内容（严格哨兵或兼容哨兵长度不同） |
|||
int matchedSentinelLength = rawText.startsWith(SENTINEL, lastIdx) ? SENTINEL.length() : LOOSE_SENTINEL.length(); |
|||
String suggestionsPart = rawText.substring(lastIdx + matchedSentinelLength).trim(); |
|||
|
|||
List<String> suggestions = parseSuggestionsJson(suggestionsPart); |
|||
return new ParsedResponse(answer, suggestions); |
|||
} |
|||
|
|||
/** |
|||
* 流式路径：从原始 Flux 中分离 answer 和 suggestions。 |
|||
* <p> |
|||
* 采用「安全缓冲区」策略防止哨兵泄漏到 UI： |
|||
* <ol> |
|||
* <li>每次 chunk 追加到累积缓冲区</li> |
|||
* <li>安全区域 = 缓冲区去掉末尾 SENTINEL.length() 字符（预留哨兵跨 chunk 截断空间）</li> |
|||
* <li>在安全区域内检查哨兵：找到则发出哨兵前的剩余内容，后续全进 suggestionsBuffer</li> |
|||
* <li>未找到哨兵则发出安全区域内容</li> |
|||
* <li>doOnComplete 时：哨兵已发现则解析 suggestions；未发现则发出安全缓冲区内剩余内容</li> |
|||
* </ol> |
|||
* |
|||
* @param rawStream LLM 原始输出流 |
|||
* @param cache 建议缓存 |
|||
* @param chatId 会话 ID |
|||
* @return 纯 answer 的 Flux（不含哨兵及之后的 suggestions） |
|||
*/ |
|||
/** |
|||
* 可变状态容器（供 lambda 内部修改）。 |
|||
*/ |
|||
private static class StreamState { |
|||
final StringBuilder buf = new StringBuilder(); |
|||
int emitted = 0; |
|||
boolean sentinelFound = false; |
|||
} |
|||
|
|||
public static Flux<String> parseFromStream(Flux<String> rawStream, SuggestionCache cache, String chatId) { |
|||
StreamState state = new StreamState(); |
|||
|
|||
return rawStream |
|||
.concatMap(chunk -> Flux.<String>create(sink -> { |
|||
if (state.sentinelFound) { |
|||
sink.complete(); |
|||
return; |
|||
} |
|||
state.buf.append(chunk); |
|||
|
|||
int maxSentinelLen = Math.max(SENTINEL.length(), LOOSE_SENTINEL.length()); |
|||
int safeEnd = Math.max(0, state.buf.length() - maxSentinelLen); |
|||
if (safeEnd <= state.emitted) { |
|||
sink.complete(); |
|||
return; |
|||
} |
|||
|
|||
String safeRegion = state.buf.substring(state.emitted, safeEnd); |
|||
int sentinelIdx = findSentinelIndex(safeRegion); |
|||
|
|||
if (sentinelIdx >= 0) { |
|||
state.sentinelFound = true; |
|||
String beforeSentinel = safeRegion.substring(0, sentinelIdx); |
|||
if (!beforeSentinel.isEmpty()) { |
|||
sink.next(beforeSentinel); |
|||
} |
|||
// 确定实际匹配到的哨兵长度 |
|||
int matchedLen = safeRegion.startsWith(SENTINEL, sentinelIdx) |
|||
? SENTINEL.length() |
|||
: LOOSE_SENTINEL.length(); |
|||
// 哨兵在安全区域内的位置 sentinelIdx，相对全缓冲区即 emitted + sentinelIdx |
|||
// emitted 推进到哨兵结束后，后续内容进 suggestions 解析 |
|||
state.emitted = state.emitted + sentinelIdx + matchedLen; |
|||
log.debug("流式路径检测到哨兵: chatId={}, pos={}, matchedLen={}", chatId, state.emitted, matchedLen); |
|||
} else { |
|||
if (!safeRegion.isEmpty()) { |
|||
sink.next(safeRegion); |
|||
} |
|||
state.emitted = safeEnd; |
|||
} |
|||
sink.complete(); |
|||
})) |
|||
// 流结束后发出安全缓冲区内未发出的残留内容（无哨兵场景） |
|||
.concatWith(Flux.defer(() -> { |
|||
if (!state.sentinelFound && state.emitted < state.buf.length()) { |
|||
String residual = state.buf.substring(state.emitted); |
|||
if (!residual.isEmpty()) { |
|||
return Flux.just(residual); |
|||
} |
|||
} |
|||
return Flux.empty(); |
|||
})) |
|||
.doOnComplete(() -> { |
|||
if (state.sentinelFound) { |
|||
String suggestionsPart = state.buf.length() > state.emitted |
|||
? state.buf.substring(state.emitted) : ""; |
|||
List<String> suggestions = parseSuggestionsJson(suggestionsPart.strip()); |
|||
if (!suggestions.isEmpty()) { |
|||
cache.put(chatId, suggestions); |
|||
log.info("流式 suggestions 解析成功: chatId={}, count={}", chatId, suggestions.size()); |
|||
} else { |
|||
log.debug("流式 suggestions 解析为空: chatId={}", chatId); |
|||
} |
|||
} |
|||
}); |
|||
} |
|||
|
|||
/** |
|||
* 容错解析 suggestions JSON 数组。 |
|||
* 先尝试标准 JSON 解析，失败则按行切分取前 3 条非空行。 |
|||
*/ |
|||
static List<String> parseSuggestionsJson(String jsonPart) { |
|||
if (jsonPart == null || jsonPart.isBlank()) { |
|||
return Collections.emptyList(); |
|||
} |
|||
|
|||
// 1. 尝试标准 JSON 解析 |
|||
String trimmed = jsonPart.strip(); |
|||
// 剥离旧版哨兵标记（___SUGGESTIONS___ 及兼容大写变形），取哨兵后的 JSON 部分。 |
|||
// 旧版 suggestion_prompt 会要求模型输出哨兵，若未剥离会被按行降级解析成垃圾条目。 |
|||
int sentinelIdx = findSentinelIndex(trimmed); |
|||
if (sentinelIdx >= 0) { |
|||
int matchedLen = trimmed.startsWith(SENTINEL, sentinelIdx) |
|||
? SENTINEL.length() |
|||
: LOOSE_SENTINEL.length(); |
|||
trimmed = trimmed.substring(sentinelIdx + matchedLen).strip(); |
|||
} |
|||
// 去掉可能的 markdown 代码块包裹 |
|||
trimmed = trimCodeBlock(trimmed); |
|||
|
|||
try { |
|||
List<String> parsed = objectMapper.readValue(trimmed, new TypeReference<List<String>>() {}); |
|||
List<String> result = new ArrayList<>(); |
|||
for (String s : parsed) { |
|||
if (s != null && !s.isBlank()) { |
|||
result.add(s.strip()); |
|||
} |
|||
} |
|||
if (!result.isEmpty()) { |
|||
return result.size() <= 3 ? result : result.subList(0, 3); |
|||
} |
|||
} catch (Exception e) { |
|||
log.debug("标准 JSON 解析 suggestions 失败，尝试按行切分: {}", e.getMessage()); |
|||
} |
|||
|
|||
// 2. 降级：按行切分，取前 3 条 |
|||
return parseByLines(trimmed); |
|||
} |
|||
|
|||
/** |
|||
* 降级解析：按行切分，去掉序号前缀后取前 3 条非空行。 |
|||
*/ |
|||
private static List<String> parseByLines(String text) { |
|||
List<String> lines = text.lines() |
|||
.map(String::strip) |
|||
.map(SuggestionResponseParser::stripNumberPrefix) |
|||
.map(SuggestionResponseParser::stripQuotes) |
|||
.filter(s -> !s.isBlank()) |
|||
.collect(Collectors.toList()); |
|||
|
|||
if (lines.isEmpty()) { |
|||
return Collections.emptyList(); |
|||
} |
|||
return lines.size() <= 3 ? lines : lines.subList(0, 3); |
|||
} |
|||
|
|||
/** 去掉可能的 markdown 代码块包裹（支持 3 个及以上反引号） */ |
|||
private static String trimCodeBlock(String s) { |
|||
if (s.startsWith("```") && s.endsWith("```")) { |
|||
// 计算开头的反引号数量 |
|||
int openCount = 0; |
|||
while (openCount < s.length() && s.charAt(openCount) == '`') openCount++; |
|||
if (openCount >= 3 && s.endsWith("`".repeat(openCount))) { |
|||
String inner = s.substring(openCount, s.length() - openCount).strip(); |
|||
if (inner.startsWith("json")) { |
|||
inner = inner.substring(4).strip(); |
|||
} else if (inner.startsWith("JSON")) { |
|||
inner = inner.substring(4).strip(); |
|||
} |
|||
return inner; |
|||
} |
|||
} |
|||
return s; |
|||
} |
|||
|
|||
/** 去掉行首序号，如 "1." "2." "3." "1、" "2、" "3、" */ |
|||
private static String stripNumberPrefix(String s) { |
|||
if (s.length() >= 2 && Character.isDigit(s.charAt(0)) && |
|||
(s.charAt(1) == '.' || s.charAt(1) == '、' || s.charAt(1) == ')')) { |
|||
return s.substring(2).strip(); |
|||
} |
|||
return s; |
|||
} |
|||
|
|||
/** 去掉首尾引号 */ |
|||
private static String stripQuotes(String s) { |
|||
if (s.length() >= 2) { |
|||
char first = s.charAt(0); |
|||
char last = s.charAt(s.length() - 1); |
|||
if ((first == '"' && last == '"') || (first == '\'' && last == '\'') || |
|||
(first == '“' && last == '”')) { // 中文引号 " " |
|||
return s.substring(1, s.length() - 1).strip(); |
|||
} |
|||
} |
|||
return s; |
|||
} |
|||
|
|||
/** |
|||
* 解析结果值对象。 |
|||
* |
|||
* @param answer LLM 回答正文（不含哨兵及之后的 suggestions） |
|||
* @param suggestions 建议问题列表（0～3 条） |
|||
*/ |
|||
public record ParsedResponse(String answer, List<String> suggestions) { |
|||
} |
|||
} |
|||
@ -1,73 +0,0 @@ |
|||
package com.wok.supportbot.document.extract; |
|||
|
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.apache.tika.Tika; |
|||
import org.apache.tika.exception.TikaException; |
|||
import org.springframework.ai.document.Document; |
|||
import org.springframework.core.io.Resource; |
|||
import org.springframework.core.io.FileSystemResource; |
|||
import org.springframework.stereotype.Component; |
|||
import org.springframework.web.multipart.MultipartFile; |
|||
|
|||
import java.io.File; |
|||
import java.io.IOException; |
|||
import java.util.Collections; |
|||
import java.util.List; |
|||
import java.util.UUID; |
|||
|
|||
@Component |
|||
@Slf4j |
|||
public class TikaDocumentReader { |
|||
|
|||
/** |
|||
* 从已保存的文件解析内容 |
|||
*/ |
|||
public List<Document> readFromFile(File file) { |
|||
try { |
|||
Tika tika = new Tika(); |
|||
String text = tika.parseToString(new java.io.FileInputStream(file)); |
|||
|
|||
Document doc = Document.builder() |
|||
.id(UUID.randomUUID().toString()) |
|||
.text(text) |
|||
.build(); |
|||
|
|||
return Collections.singletonList(doc); |
|||
|
|||
} catch (IOException | TikaException e) { |
|||
log.error("Tika 文件解析失败", e); |
|||
throw new RuntimeException("Tika 文件解析失败", e); |
|||
} |
|||
} |
|||
|
|||
/** |
|||
* 从 MultipartFile 解析内容（兼容旧接口） |
|||
*/ |
|||
public List<Document> read(MultipartFile file) { |
|||
try { |
|||
Tika tika = new Tika(); |
|||
String text = tika.parseToString(file.getInputStream()); |
|||
|
|||
Document doc = Document.builder() |
|||
.id(UUID.randomUUID().toString()) |
|||
.text(text) |
|||
.build(); |
|||
|
|||
return Collections.singletonList(doc); |
|||
|
|||
} catch (IOException | TikaException e) { |
|||
log.error("Tika 文件解析失败", e); |
|||
throw new RuntimeException("Tika 文件解析失败", e); |
|||
} |
|||
} |
|||
|
|||
/** |
|||
* 获取文件扩展名（含点号），如 ".pptx"。无扩展名时返回 ".tmp" |
|||
*/ |
|||
private String getExtension(String filename) { |
|||
if (filename == null || !filename.contains(".")) { |
|||
return ".tmp"; |
|||
} |
|||
return filename.substring(filename.lastIndexOf(".")); |
|||
} |
|||
} |
|||
@ -1,67 +0,0 @@ |
|||
package com.wok.supportbot.document.transform; |
|||
|
|||
import com.wok.supportbot.config.ChunkConfig; |
|||
import org.springframework.ai.document.Document; |
|||
import org.springframework.ai.transformer.splitter.TokenTextSplitter; |
|||
import org.springframework.beans.factory.annotation.Autowired; |
|||
import org.springframework.stereotype.Component; |
|||
|
|||
import java.util.List; |
|||
|
|||
/** |
|||
* 自定义基于 Token 的切词器 |
|||
* 支持通过 ChunkConfig 动态调整分块参数 |
|||
*/ |
|||
@Component |
|||
public class MyTokenTextSplitter { |
|||
|
|||
@Autowired |
|||
private ChunkConfig chunkConfig; |
|||
|
|||
/** |
|||
* 使用全局配置参数创建分割器 |
|||
*/ |
|||
public List<Document> splitDocuments(List<Document> documents) { |
|||
TokenTextSplitter splitter = new TokenTextSplitter( |
|||
chunkConfig.getChunkSize(), |
|||
chunkConfig.getOverlap(), |
|||
chunkConfig.getMinChunkSizeChars(), |
|||
chunkConfig.getMaxNumChunks(), |
|||
chunkConfig.isKeepSeparator() |
|||
); |
|||
return splitter.apply(documents); |
|||
} |
|||
|
|||
/** |
|||
* 使用自定义参数创建分割器（覆盖全局配置） |
|||
* |
|||
* @param documents 文档列表 |
|||
* @param chunkSize 分块大小 |
|||
* @param overlap 重叠大小 |
|||
*/ |
|||
public List<Document> splitDocuments(List<Document> documents, Integer chunkSize, Integer overlap) { |
|||
int cs = chunkSize != null ? chunkSize : chunkConfig.getChunkSize(); |
|||
int ol = overlap != null ? overlap : chunkConfig.getOverlap(); |
|||
TokenTextSplitter splitter = new TokenTextSplitter( |
|||
cs, ol, |
|||
chunkConfig.getMinChunkSizeChars(), |
|||
chunkConfig.getMaxNumChunks(), |
|||
chunkConfig.isKeepSeparator() |
|||
); |
|||
return splitter.apply(documents); |
|||
} |
|||
|
|||
/** |
|||
* 使用自定义参数创建分割器（全参数覆盖） |
|||
*/ |
|||
public List<Document> splitCustomized(List<Document> documents) { |
|||
TokenTextSplitter splitter = new TokenTextSplitter( |
|||
chunkConfig.getChunkSize(), |
|||
chunkConfig.getOverlap(), |
|||
chunkConfig.getMinChunkSizeChars(), |
|||
chunkConfig.getMaxNumChunks(), |
|||
chunkConfig.isKeepSeparator() |
|||
); |
|||
return splitter.apply(documents); |
|||
} |
|||
} |
|||
@ -0,0 +1,252 @@ |
|||
package com.wok.supportbot.document.transform; |
|||
|
|||
import com.knuddels.jtokkit.Encodings; |
|||
import com.knuddels.jtokkit.api.Encoding; |
|||
import com.knuddels.jtokkit.api.EncodingType; |
|||
import com.knuddels.jtokkit.api.IntArrayList; |
|||
import org.slf4j.Logger; |
|||
import org.slf4j.LoggerFactory; |
|||
import org.springframework.ai.transformer.splitter.TextSplitter; |
|||
import org.springframework.util.Assert; |
|||
|
|||
import java.util.ArrayList; |
|||
import java.util.List; |
|||
|
|||
/** |
|||
* 带重叠（overlap）的 Token 分块器。 |
|||
* |
|||
* <p>Spring AI 的 {@link org.springframework.ai.transformer.splitter.TokenTextSplitter} |
|||
* 不支持 overlap —— 它的 5 参构造器与 Builder 均无该形参（社区 PR #4054 已标记不向 1.x 回迁）。 |
|||
* 本类直接继承标准基类 {@link TextSplitter}，复刻 TokenTextSplitter 的全部切分语义 |
|||
* （标点截断、最小分块长度、分块数上限、分隔符处理），仅把「前进步长」由 chunkSize |
|||
* 改为 {@code chunkSize - overlap}，从而让重叠真正生效。 |
|||
* |
|||
* <p>元数据继承（{@code parent_document_id} / {@code chunk_index} / {@code total_chunks}） |
|||
* 由 {@link TextSplitter} 基类统一处理，本类只负责文本切分。 |
|||
* |
|||
* <p>Token 编码与 TokenTextSplitter 保持一致：jtokkit + CL100K_BASE。 |
|||
*/ |
|||
public class OverlapTokenTextSplitter extends TextSplitter { |
|||
|
|||
private static final Logger log = LoggerFactory.getLogger(OverlapTokenTextSplitter.class); |
|||
|
|||
/** 默认分块大小（token 数） */ |
|||
private static final int DEFAULT_CHUNK_SIZE = 800; |
|||
|
|||
/** 默认重叠 token 数（0 表示不重叠，等价于 TokenTextSplitter 行为） */ |
|||
private static final int DEFAULT_OVERLAP = 0; |
|||
|
|||
/** 默认最小分块字符数 */ |
|||
private static final int MIN_CHUNK_SIZE_CHARS = 350; |
|||
|
|||
/** 默认丢弃阈值：长度不超过该值的分块不入库 */ |
|||
private static final int MIN_CHUNK_LENGTH_TO_EMBED = 5; |
|||
|
|||
/** 默认单篇文档最大分块数 */ |
|||
private static final int MAX_NUM_CHUNKS = 10000; |
|||
|
|||
/** 默认保留分隔符 */ |
|||
private static final boolean KEEP_SEPARATOR = true; |
|||
|
|||
private final Encoding encoding = Encodings.newLazyEncodingRegistry().getEncoding(EncodingType.CL100K_BASE); |
|||
|
|||
/** 目标分块大小（token 数，不含重叠部分） */ |
|||
private final int chunkSize; |
|||
|
|||
/** 相邻分块的重叠 token 数 */ |
|||
private final int overlap; |
|||
|
|||
/** 最小分块字符数：仅当剩余 token 数超过 chunkSize 时才按标点截断，且截断点须大于该值 */ |
|||
private final int minChunkSizeChars; |
|||
|
|||
/** 丢弃长度不超过该值的分块 */ |
|||
private final int minChunkLengthToEmbed; |
|||
|
|||
/** 单篇文档最大分块数 */ |
|||
private final int maxNumChunks; |
|||
|
|||
/** 是否保留分隔符（false 时把换行替换为空格） */ |
|||
private final boolean keepSeparator; |
|||
|
|||
private OverlapTokenTextSplitter(int chunkSize, int overlap, int minChunkSizeChars, |
|||
int minChunkLengthToEmbed, int maxNumChunks, boolean keepSeparator) { |
|||
Assert.isTrue(chunkSize > 0, "chunkSize 必须大于 0"); |
|||
this.chunkSize = chunkSize; |
|||
// 重叠必须严格小于分块大小，否则前进步长会 <= 0 导致死循环；此处收敛到合法区间而非直接报错 |
|||
int clamped = Math.max(0, Math.min(overlap, chunkSize - 1)); |
|||
if (clamped != overlap) { |
|||
log.warn("overlap({}) 超出合法区间 [0, chunkSize-1={}]，已收敛为 {}；" |
|||
+ "该配置会使前进步长退化为 1 个 token，产生大量分块，请检查 knowledge.chunk.overlap", |
|||
overlap, chunkSize - 1, clamped); |
|||
} |
|||
this.overlap = clamped; |
|||
this.minChunkSizeChars = minChunkSizeChars; |
|||
this.minChunkLengthToEmbed = minChunkLengthToEmbed; |
|||
this.maxNumChunks = maxNumChunks; |
|||
this.keepSeparator = keepSeparator; |
|||
} |
|||
|
|||
public static Builder builder() { |
|||
return new Builder(); |
|||
} |
|||
|
|||
/** |
|||
* 标点截断允许的最小前进步长（token 数）。 |
|||
* <p> |
|||
* 防止「标点截断」与「overlap」叠加后把前进步长压到 1 个 token 导致分块数爆炸： |
|||
* 截断后必须仍能前进这么多 token，否则放弃本次截断（宁可切断句子也要保证分块数量可控）。 |
|||
* <p> |
|||
* overlap=0 时返回 1，即允许任意截断，与标准 {@code TokenTextSplitter} 行为完全一致。 |
|||
*/ |
|||
private int minAdvance() { |
|||
if (this.overlap == 0) { |
|||
return 1; |
|||
} |
|||
return Math.max(1, (this.chunkSize - this.overlap) / 2); |
|||
} |
|||
|
|||
@Override |
|||
protected List<String> splitText(String text) { |
|||
if (text == null || text.trim().isEmpty()) { |
|||
return new ArrayList<>(); |
|||
} |
|||
|
|||
List<Integer> tokens = getEncodedTokens(text); |
|||
List<String> chunks = new ArrayList<>(); |
|||
int numChunks = 0; |
|||
|
|||
while (!tokens.isEmpty() && numChunks < this.maxNumChunks) { |
|||
int windowSize = Math.min(this.chunkSize, tokens.size()); |
|||
boolean lastWindow = windowSize >= tokens.size(); |
|||
String chunkText = decodeTokens(tokens.subList(0, windowSize)); |
|||
|
|||
// 空白块直接跳过（不产出，也不做重叠回退） |
|||
if (chunkText.trim().isEmpty()) { |
|||
tokens = tokens.subList(windowSize, tokens.size()); |
|||
continue; |
|||
} |
|||
|
|||
// 仅当剩余 token 数超过 chunkSize 时才做标点截断，避免小文本被无谓切分 |
|||
if (tokens.size() > this.chunkSize) { |
|||
int lastPunctuation = Math.max(chunkText.lastIndexOf('.'), |
|||
Math.max(chunkText.lastIndexOf('?'), |
|||
Math.max(chunkText.lastIndexOf('!'), chunkText.lastIndexOf('\n')))); |
|||
if (lastPunctuation != -1 && lastPunctuation > this.minChunkSizeChars) { |
|||
String candidate = chunkText.substring(0, lastPunctuation + 1); |
|||
// 前进步长 = 截断后消耗的 token 数 - overlap。截断会缩短消耗量， |
|||
// 若不加约束，截断点靠前时步长会被压到 1 个 token，分块数成倍膨胀 |
|||
// （实测 chunkSize=60/overlap=30 时可达 10 倍）。此处要求截断后仍能前进至少 minAdvance()。 |
|||
if (getEncodedTokens(candidate).size() - this.overlap >= minAdvance()) { |
|||
chunkText = candidate; |
|||
} |
|||
} |
|||
} |
|||
|
|||
String chunkTextToAppend = this.keepSeparator |
|||
? chunkText.trim() |
|||
: chunkText.replace(System.lineSeparator(), " ").trim(); |
|||
if (chunkTextToAppend.length() > this.minChunkLengthToEmbed) { |
|||
chunks.add(chunkTextToAppend); |
|||
} |
|||
numChunks++; |
|||
|
|||
// 本窗口已覆盖全部剩余 token，无需再产出重叠块 |
|||
if (lastWindow) { |
|||
tokens = new ArrayList<>(); |
|||
break; |
|||
} |
|||
|
|||
// 本块实际消耗的 token 数（可能因标点截断而少于窗口大小） |
|||
int consumed = getEncodedTokens(chunkText).size(); |
|||
// 前进步长 = 消耗量 - 重叠量；至少前进 1 个 token，避免死循环 |
|||
int step = Math.max(1, consumed - this.overlap); |
|||
tokens = tokens.subList(step, tokens.size()); |
|||
} |
|||
|
|||
// 处理剩余 token（与 TokenTextSplitter 保持一致，用于达到 maxNumChunks 上限的场景） |
|||
if (!tokens.isEmpty()) { |
|||
String remainingText = decodeTokens(tokens).replace(System.lineSeparator(), " ").trim(); |
|||
if (remainingText.length() > this.minChunkLengthToEmbed) { |
|||
chunks.add(remainingText); |
|||
} |
|||
} |
|||
|
|||
return chunks; |
|||
} |
|||
|
|||
private List<Integer> getEncodedTokens(String text) { |
|||
Assert.notNull(text, "Text must not be null"); |
|||
return this.encoding.encode(text).boxed(); |
|||
} |
|||
|
|||
private String decodeTokens(List<Integer> tokens) { |
|||
Assert.notNull(tokens, "Tokens must not be null"); |
|||
IntArrayList tokensIntArray = new IntArrayList(tokens.size()); |
|||
tokens.forEach(tokensIntArray::add); |
|||
return this.encoding.decode(tokensIntArray); |
|||
} |
|||
|
|||
/** |
|||
* 分块器构建器 |
|||
*/ |
|||
public static final class Builder { |
|||
|
|||
private int chunkSize = DEFAULT_CHUNK_SIZE; |
|||
|
|||
private int overlap = DEFAULT_OVERLAP; |
|||
|
|||
private int minChunkSizeChars = MIN_CHUNK_SIZE_CHARS; |
|||
|
|||
private int minChunkLengthToEmbed = MIN_CHUNK_LENGTH_TO_EMBED; |
|||
|
|||
private int maxNumChunks = MAX_NUM_CHUNKS; |
|||
|
|||
private boolean keepSeparator = KEEP_SEPARATOR; |
|||
|
|||
private Builder() { |
|||
} |
|||
|
|||
/** 目标分块大小（token 数） */ |
|||
public Builder withChunkSize(int chunkSize) { |
|||
this.chunkSize = chunkSize; |
|||
return this; |
|||
} |
|||
|
|||
/** 相邻分块的重叠 token 数 */ |
|||
public Builder withOverlap(int overlap) { |
|||
this.overlap = overlap; |
|||
return this; |
|||
} |
|||
|
|||
/** 最小分块字符数（标点截断的下限） */ |
|||
public Builder withMinChunkSizeChars(int minChunkSizeChars) { |
|||
this.minChunkSizeChars = minChunkSizeChars; |
|||
return this; |
|||
} |
|||
|
|||
/** 丢弃长度不超过该值的分块 */ |
|||
public Builder withMinChunkLengthToEmbed(int minChunkLengthToEmbed) { |
|||
this.minChunkLengthToEmbed = minChunkLengthToEmbed; |
|||
return this; |
|||
} |
|||
|
|||
/** 单篇文档最大分块数 */ |
|||
public Builder withMaxNumChunks(int maxNumChunks) { |
|||
this.maxNumChunks = maxNumChunks; |
|||
return this; |
|||
} |
|||
|
|||
/** 是否保留分隔符 */ |
|||
public Builder withKeepSeparator(boolean keepSeparator) { |
|||
this.keepSeparator = keepSeparator; |
|||
return this; |
|||
} |
|||
|
|||
public OverlapTokenTextSplitter build() { |
|||
return new OverlapTokenTextSplitter(this.chunkSize, this.overlap, this.minChunkSizeChars, |
|||
this.minChunkLengthToEmbed, this.maxNumChunks, this.keepSeparator); |
|||
} |
|||
|
|||
} |
|||
|
|||
} |
|||
@ -1,73 +0,0 @@ |
|||
package com.wok.supportbot.mcp; |
|||
|
|||
import org.springframework.http.codec.ServerSentEvent; |
|||
|
|||
/** |
|||
* SSE 事件构建器 |
|||
* 用于在 MCP 工具调用流程中构建标准 SSE 事件， |
|||
* 让前端能区分文本内容与工具调用状态。 |
|||
* |
|||
* 事件类型： |
|||
* - message: 普通文本内容 |
|||
* - tool_call_start: 工具调用开始 |
|||
* - tool_call_result: 工具调用结果返回 |
|||
* - error: 错误信息 |
|||
*/ |
|||
public class SseEventBuilder { |
|||
|
|||
/** |
|||
* 构建普通文本消息事件 |
|||
*/ |
|||
public static ServerSentEvent<String> messageEvent(String data) { |
|||
return ServerSentEvent.<String>builder() |
|||
.event("message") |
|||
.data(data) |
|||
.build(); |
|||
} |
|||
|
|||
/** |
|||
* 构建工具调用开始事件 |
|||
*/ |
|||
public static ServerSentEvent<String> toolCallStartEvent(String toolName, String input) { |
|||
String json = String.format("{\"tool\":\"%s\",\"input\":\"%s\"}", |
|||
escapeJson(toolName), escapeJson(input)); |
|||
return ServerSentEvent.<String>builder() |
|||
.event("tool_call_start") |
|||
.data(json) |
|||
.build(); |
|||
} |
|||
|
|||
/** |
|||
* 构建工具调用结果事件 |
|||
*/ |
|||
public static ServerSentEvent<String> toolCallResultEvent(String toolName, String result, long latencyMs) { |
|||
String json = String.format("{\"tool\":\"%s\",\"result\":\"%s\",\"latencyMs\":%d}", |
|||
escapeJson(toolName), escapeJson(result), latencyMs); |
|||
return ServerSentEvent.<String>builder() |
|||
.event("tool_call_result") |
|||
.data(json) |
|||
.build(); |
|||
} |
|||
|
|||
/** |
|||
* 构建错误事件 |
|||
*/ |
|||
public static ServerSentEvent<String> errorEvent(String message) { |
|||
return ServerSentEvent.<String>builder() |
|||
.event("error") |
|||
.data("{\"message\":\"" + escapeJson(message) + "\"}") |
|||
.build(); |
|||
} |
|||
|
|||
/** |
|||
* JSON 字符串转义 |
|||
*/ |
|||
private static String escapeJson(String s) { |
|||
if (s == null) return ""; |
|||
return s.replace("\\", "\\\\") |
|||
.replace("\"", "\\\"") |
|||
.replace("\n", "\\n") |
|||
.replace("\r", "\\r") |
|||
.replace("\t", "\\t"); |
|||
} |
|||
} |
|||
Write
Preview
Loading…
Cancel
Save
Reference in new issue