135 changed files with 5283 additions and 1907 deletions
-
116CLAUDE.md
-
2DEPLOY.md
-
40README.md
-
70client/src/api.ts
-
132client/src/chat.ts
-
1client/src/config.ts
-
28client/src/dom.ts
-
2client/src/i18n.ts
-
1client/src/index.ts
-
20client/src/types.ts
-
10frontend/UI-DEV-GUIDE.md
-
5frontend/components.d.ts
-
39frontend/src/App.vue
-
7frontend/src/api/api-key.ts
-
16frontend/src/api/auth.ts
-
4frontend/src/api/conversation.ts
-
19frontend/src/api/document.ts
-
4frontend/src/api/faq.ts
-
17frontend/src/api/folder.ts
-
4frontend/src/api/llm-trace.ts
-
9frontend/src/api/mcp-server.ts
-
4frontend/src/api/model-config.ts
-
76frontend/src/api/request.ts
-
8frontend/src/api/sensitive-word.ts
-
39frontend/src/api/upload.ts
-
4frontend/src/api/user.ts
-
7frontend/src/api/webhook.ts
-
40frontend/src/components/BaseTable.vue
-
4frontend/src/layouts/MainLayout.vue
-
91frontend/src/layouts/Topbar.vue
-
1frontend/src/router/index.ts
-
18frontend/src/stores/auth.ts
-
14frontend/src/stores/document.ts
-
7frontend/src/styles/shared.css
-
21frontend/src/types/models.ts
-
4frontend/src/utils/sse.ts
-
21frontend/src/utils/token.ts
-
11frontend/src/views/AccountManager.vue
-
14frontend/src/views/ApiKeyManager.vue
-
12frontend/src/views/AuditLogManager.vue
-
25frontend/src/views/CategoryManager.vue
-
16frontend/src/views/ConversationManager.vue
-
6frontend/src/views/DashboardPanel.vue
-
11frontend/src/views/DocDetail.vue
-
588frontend/src/views/DocList.vue
-
262frontend/src/views/DocUpload.vue
-
18frontend/src/views/FaqManager.vue
-
18frontend/src/views/FeedbackOps.vue
-
8frontend/src/views/LoginPage.vue
-
44frontend/src/views/McpServerManager.vue
-
70frontend/src/views/ModelConfigManager.vue
-
42frontend/src/views/PipelineFlow.vue
-
52frontend/src/views/PromptTracePanel.vue
-
20frontend/src/views/SensitiveWordManager.vue
-
19frontend/src/views/UserManager.vue
-
12frontend/src/views/WebhookManager.vue
-
65pom.xml
-
5src/main/java/com/wok/supportbot/SupportBotApplication.java
-
43src/main/java/com/wok/supportbot/advisor/ReReadingAdvisor.java
-
13src/main/java/com/wok/supportbot/app/AssistantApp.java
-
32src/main/java/com/wok/supportbot/app/ChatPipeline.java
-
2src/main/java/com/wok/supportbot/app/ChatRequest.java
-
5src/main/java/com/wok/supportbot/app/ChatResult.java
-
31src/main/java/com/wok/supportbot/app/SuggestionGenerator.java
-
285src/main/java/com/wok/supportbot/app/SuggestionResponseParser.java
-
219src/main/java/com/wok/supportbot/auth/AuthController.java
-
85src/main/java/com/wok/supportbot/chatmemory/FileBasedChatMemory.java
-
40src/main/java/com/wok/supportbot/common/SortUtils.java
-
36src/main/java/com/wok/supportbot/config/AsyncExecutorConfig.java
-
8src/main/java/com/wok/supportbot/config/ChatModelFactory.java
-
31src/main/java/com/wok/supportbot/config/CorsConfig.java
-
158src/main/java/com/wok/supportbot/config/DatabaseInitConfig.java
-
30src/main/java/com/wok/supportbot/config/EmbeddingModelFactory.java
-
16src/main/java/com/wok/supportbot/config/VolcengineMultimodalEmbeddingModel.java
-
6src/main/java/com/wok/supportbot/controller/AiModelConfigController.java
-
6src/main/java/com/wok/supportbot/controller/ApiKeyController.java
-
6src/main/java/com/wok/supportbot/controller/AttachmentController.java
-
12src/main/java/com/wok/supportbot/controller/AuthController.java
-
6src/main/java/com/wok/supportbot/controller/ConversationController.java
-
143src/main/java/com/wok/supportbot/controller/DocumentController.java
-
6src/main/java/com/wok/supportbot/controller/FaqController.java
-
136src/main/java/com/wok/supportbot/controller/FolderController.java
-
6src/main/java/com/wok/supportbot/controller/LlmCallTraceController.java
-
33src/main/java/com/wok/supportbot/controller/McpServerConfigController.java
-
6src/main/java/com/wok/supportbot/controller/MessageFeedbackController.java
-
22src/main/java/com/wok/supportbot/controller/SensitiveWordController.java
-
38src/main/java/com/wok/supportbot/controller/SysUserController.java
-
6src/main/java/com/wok/supportbot/controller/WebhookController.java
-
83src/main/java/com/wok/supportbot/converter/MessageConverter.java
-
12src/main/java/com/wok/supportbot/dao/KnowledgeFolderMapper.java
-
73src/main/java/com/wok/supportbot/document/extract/TikaDocumentReader.java
-
30src/main/java/com/wok/supportbot/document/transform/MyKeywordEnricher.java
-
67src/main/java/com/wok/supportbot/document/transform/MyTokenTextSplitter.java
-
252src/main/java/com/wok/supportbot/document/transform/OverlapTokenTextSplitter.java
-
63src/main/java/com/wok/supportbot/entity/FolderNode.java
-
15src/main/java/com/wok/supportbot/entity/KnowledgeDocument.java
-
80src/main/java/com/wok/supportbot/entity/KnowledgeFolder.java
-
4src/main/java/com/wok/supportbot/entity/SysUser.java
-
5src/main/java/com/wok/supportbot/handler/MyMetaObjectHandler.java
-
36src/main/java/com/wok/supportbot/mcp/McpToolCallback.java
@ -0,0 +1,17 @@ |
|||
import request from './request' |
|||
import type { ApiResponse } from '@/types/api' |
|||
|
|||
/** 获取目录树（可按分类过滤） */ |
|||
export function getFolderTree(categoryId?: string): Promise<ApiResponse> { return request.get(`/folder/tree${categoryId ? `?categoryId=${categoryId}` : ''}`).then(r => r.data) } |
|||
|
|||
/** 获取目录列表（可按分类过滤） */ |
|||
export function getFolderList(categoryId?: string): Promise<ApiResponse> { return request.get(`/folder/list${categoryId ? `?categoryId=${categoryId}` : ''}`).then(r => r.data) } |
|||
|
|||
/** 创建目录 */ |
|||
export function createFolder(data: any): Promise<ApiResponse> { return request.post('/folder', data).then(r => r.data) } |
|||
|
|||
/** 更新目录 */ |
|||
export function updateFolder(id: string, data: any): Promise<ApiResponse> { return request.put(`/folder/${id}`, data).then(r => r.data) } |
|||
|
|||
/** 删除目录 */ |
|||
export function deleteFolder(id: string): Promise<ApiResponse> { return request.delete(`/folder/${id}`).then(r => r.data) } |
|||
@ -1,12 +1,17 @@ |
|||
import request from './request' |
|||
import type { ApiResponse } from '@/types/api' |
|||
|
|||
export function listMcpServers(page = 1, size = 10): Promise<ApiResponse> { return request.get(`/mcp-server/list?page=${page}&size=${size}`).then(r => r.data) } |
|||
export function listMcpServers(page = 1, size = 10, sortField?: string, sortOrder?: string): Promise<ApiResponse> { |
|||
let path = `/mcp-server/list?page=${page}&size=${size}` |
|||
if (sortField) path += `&sortField=${sortField}` |
|||
if (sortOrder) path += `&sortOrder=${sortOrder}` |
|||
return request.get(path).then(r => r.data) |
|||
} |
|||
export function getMcpServerDetail(id: string): Promise<ApiResponse> { return request.get(`/mcp-server/${id}`).then(r => r.data) } |
|||
export function createMcpServer(data: any): Promise<ApiResponse> { return request.post('/mcp-server', data).then(r => r.data) } |
|||
export function updateMcpServer(id: string, data: any): Promise<ApiResponse> { return request.put(`/mcp-server/${id}`, data).then(r => r.data) } |
|||
export function deleteMcpServer(id: string): Promise<ApiResponse> { return request.delete(`/mcp-server/${id}`).then(r => r.data) } |
|||
export function toggleMcpServer(id: string, active: boolean): Promise<ApiResponse> { return request.put(`/mcp-server/${id}/toggle?active=${active}`).then(r => r.data) } |
|||
export function toggleMcpServer(id: string, active: boolean): Promise<ApiResponse> { return request.put(`/mcp-server/${id}/toggle`, { active }).then(r => r.data) } |
|||
export function testMcpServer(id: string): Promise<ApiResponse> { return request.post(`/mcp-server/${id}/test`, {}).then(r => r.data) } |
|||
export function refreshMcpServers(): Promise<ApiResponse> { return request.post('/mcp-server/refresh', {}).then(r => r.data) } |
|||
export function listAvailableMcpTools(): Promise<ApiResponse> { return request.get('/mcp-server/tools').then(r => r.data) } |
|||
@ -1,7 +1,12 @@ |
|||
import request from './request' |
|||
import type { ApiResponse } from '@/types/api' |
|||
|
|||
export function listWebhooks(page = 1, size = 20): Promise<ApiResponse> { return request.get(`/webhook/list?page=${page}&size=${size}`).then(r => r.data) } |
|||
export function listWebhooks(page = 1, size = 20, sortField?: string, sortOrder?: string): Promise<ApiResponse> { |
|||
let path = `/webhook/list?page=${page}&size=${size}` |
|||
if (sortField) path += `&sortField=${sortField}` |
|||
if (sortOrder) path += `&sortOrder=${sortOrder}` |
|||
return request.get(path).then(r => r.data) |
|||
} |
|||
export function createWebhook(data: any): Promise<ApiResponse> { return request.post('/webhook', data).then(r => r.data) } |
|||
export function updateWebhook(id: string, data: any): Promise<ApiResponse> { return request.put(`/webhook/${id}`, data).then(r => r.data) } |
|||
export function deleteWebhook(id: string): Promise<ApiResponse> { return request.delete(`/webhook/${id}`).then(r => r.data) } |
|||
|
|||
@ -0,0 +1,40 @@ |
|||
<script setup lang="ts"> |
|||
import { computed } from 'vue' |
|||
|
|||
defineOptions({ inheritAttrs: false }) |
|||
|
|||
const props = withDefaults( |
|||
defineProps<{ |
|||
columns?: any[] |
|||
/** 统一最大高度，传 '' 或 0 关闭固定表头 */ |
|||
maxHeight?: string | number |
|||
}>(), |
|||
{ |
|||
columns: () => [], |
|||
maxHeight: 'calc(100vh - 240px)', |
|||
}, |
|||
) |
|||
|
|||
// 操作列自动右固定：colKey === 'op' 且未显式 fixed 时补 fixed:'right' |
|||
const resolvedColumns = computed(() => |
|||
props.columns.map((col) => |
|||
col && col.colKey === 'op' && !col.fixed ? { ...col, fixed: 'right' } : col, |
|||
), |
|||
) |
|||
</script> |
|||
|
|||
<template> |
|||
<t-table |
|||
v-bind="$attrs" |
|||
size="small" |
|||
hover |
|||
table-layout="fixed" |
|||
:max-height="maxHeight" |
|||
:columns="resolvedColumns" |
|||
> |
|||
<!-- 动态透传所有具名插槽（op / title / fileType / ... 及作用域插槽） --> |
|||
<template v-for="(_, name) in $slots" #[name]="slotProps"> |
|||
<slot :name="name" v-bind="slotProps || {}" /> |
|||
</template> |
|||
</t-table> |
|||
</template> |
|||
@ -1,43 +0,0 @@ |
|||
package com.wok.supportbot.advisor; |
|||
|
|||
import org.springframework.ai.chat.client.ChatClientRequest; |
|||
import org.springframework.ai.chat.client.ChatClientResponse; |
|||
import org.springframework.ai.chat.client.advisor.api.AdvisorChain; |
|||
import org.springframework.ai.chat.client.advisor.api.BaseAdvisor; |
|||
|
|||
import java.util.HashMap; |
|||
import java.util.Map; |
|||
|
|||
/** |
|||
* 自定义 Re2 Advisor（适配 Spring AI 1.0.1 新 Advisor API） |
|||
* 可提高大型语言模型的推理能力 |
|||
*/ |
|||
public class ReReadingAdvisor implements BaseAdvisor { |
|||
|
|||
@Override |
|||
public String getName() { |
|||
return this.getClass().getSimpleName(); |
|||
} |
|||
|
|||
@Override |
|||
public int getOrder() { |
|||
return 0; |
|||
} |
|||
|
|||
@Override |
|||
public ChatClientRequest before(ChatClientRequest request, AdvisorChain chain) { |
|||
// Re2 策略：将用户问题重复一次以增强推理 |
|||
// 通过 context 传递原始查询，在 prompt 中追加重复指令 |
|||
Map<String, Object> newContext = new HashMap<>(request.context()); |
|||
newContext.put("re2_enabled", true); |
|||
return ChatClientRequest.builder() |
|||
.prompt(request.prompt()) |
|||
.context(newContext) |
|||
.build(); |
|||
} |
|||
|
|||
@Override |
|||
public ChatClientResponse after(ChatClientResponse response, AdvisorChain chain) { |
|||
return response; |
|||
} |
|||
} |
|||
@ -1,285 +0,0 @@ |
|||
package com.wok.supportbot.app; |
|||
|
|||
import com.fasterxml.jackson.core.type.TypeReference; |
|||
import com.fasterxml.jackson.databind.ObjectMapper; |
|||
import com.wok.supportbot.cache.SuggestionCache; |
|||
import lombok.extern.slf4j.Slf4j; |
|||
import reactor.core.publisher.Flux; |
|||
|
|||
import java.util.ArrayList; |
|||
import java.util.Collections; |
|||
import java.util.List; |
|||
import java.util.stream.Collectors; |
|||
|
|||
/** |
|||
* 建议问题解析器：从 LLM 原始输出中分离 answer 和 suggestions。 |
|||
* <p> |
|||
* 哨兵标记为 <code>___SUGGESTIONS___</code>，之后为 JSON 字符串数组。 |
|||
* 支持同步路径（直接分割完整文本）和流式路径（滑动窗口检测哨兵）。 |
|||
*/ |
|||
@Slf4j |
|||
public final class SuggestionResponseParser { |
|||
|
|||
/** 主哨兵：严格匹配提示词要求的下划线分隔标记 */ |
|||
private static final String SENTINEL = "___SUGGESTIONS___"; |
|||
|
|||
/** 兼容哨兵：模型偶尔不按指令输出时的大写变形 */ |
|||
private static final String LOOSE_SENTINEL = "SUGGESTIONS"; |
|||
|
|||
private static final ObjectMapper objectMapper = new ObjectMapper(); |
|||
|
|||
private SuggestionResponseParser() { |
|||
} |
|||
|
|||
/** |
|||
* 查找最佳哨兵位置：优先严格哨兵，其次兼容哨兵。 |
|||
* <p> |
|||
* 兼容哨兵必须后面紧跟 JSON 数组特征（'[' 或空白 + '['）， |
|||
* 避免正文中出现 "SUGGESTIONS" 普通单词时误触发。 |
|||
* |
|||
* @param text 要搜索的文本 |
|||
* @return 哨兵起始位置，未找到返回 -1 |
|||
*/ |
|||
private static int findSentinelIndex(String text) { |
|||
int strictIdx = text.lastIndexOf(SENTINEL); |
|||
if (strictIdx >= 0) { |
|||
return strictIdx; |
|||
} |
|||
|
|||
// 兼容模式：从后往前找大写 SUGGESTIONS，且后续需连接 JSON 数组 |
|||
int idx = text.lastIndexOf(LOOSE_SENTINEL); |
|||
while (idx >= 0) { |
|||
int after = idx + LOOSE_SENTINEL.length(); |
|||
if (after < text.length()) { |
|||
char c = text.charAt(after); |
|||
// 允许 SUGGESTIONS[...]、SUGGESTIONS [...]、SUGGESTIONS:\n[...] |
|||
if (c == '[' || Character.isWhitespace(c) || c == ':' || c == '-') { |
|||
return idx; |
|||
} |
|||
} |
|||
// 继续向前查找更早的兼容哨兵 |
|||
idx = text.lastIndexOf(LOOSE_SENTINEL, idx - 1); |
|||
} |
|||
return -1; |
|||
} |
|||
|
|||
/** |
|||
* 同步路径：按最后一次出现哨兵的位置分割完整文本。 |
|||
* |
|||
* @param rawText LLM 原始输出 |
|||
* @return 解析结果（answer + suggestions） |
|||
*/ |
|||
public static ParsedResponse parse(String rawText) { |
|||
if (rawText == null || rawText.isEmpty()) { |
|||
return new ParsedResponse(rawText != null ? rawText : "", Collections.emptyList()); |
|||
} |
|||
|
|||
int lastIdx = findSentinelIndex(rawText); |
|||
if (lastIdx < 0) { |
|||
// 无哨兵标记，整段作为 answer |
|||
return new ParsedResponse(rawText, Collections.emptyList()); |
|||
} |
|||
|
|||
String answer = rawText.substring(0, lastIdx).trim(); |
|||
// 按实际匹配的哨兵长度截取后续内容（严格哨兵或兼容哨兵长度不同） |
|||
int matchedSentinelLength = rawText.startsWith(SENTINEL, lastIdx) ? SENTINEL.length() : LOOSE_SENTINEL.length(); |
|||
String suggestionsPart = rawText.substring(lastIdx + matchedSentinelLength).trim(); |
|||
|
|||
List<String> suggestions = parseSuggestionsJson(suggestionsPart); |
|||
return new ParsedResponse(answer, suggestions); |
|||
} |
|||
|
|||
/** |
|||
* 流式路径：从原始 Flux 中分离 answer 和 suggestions。 |
|||
* <p> |
|||
* 采用「安全缓冲区」策略防止哨兵泄漏到 UI： |
|||
* <ol> |
|||
* <li>每次 chunk 追加到累积缓冲区</li> |
|||
* <li>安全区域 = 缓冲区去掉末尾 SENTINEL.length() 字符（预留哨兵跨 chunk 截断空间）</li> |
|||
* <li>在安全区域内检查哨兵：找到则发出哨兵前的剩余内容，后续全进 suggestionsBuffer</li> |
|||
* <li>未找到哨兵则发出安全区域内容</li> |
|||
* <li>doOnComplete 时：哨兵已发现则解析 suggestions；未发现则发出安全缓冲区内剩余内容</li> |
|||
* </ol> |
|||
* |
|||
* @param rawStream LLM 原始输出流 |
|||
* @param cache 建议缓存 |
|||
* @param chatId 会话 ID |
|||
* @return 纯 answer 的 Flux（不含哨兵及之后的 suggestions） |
|||
*/ |
|||
/** |
|||
* 可变状态容器（供 lambda 内部修改）。 |
|||
*/ |
|||
private static class StreamState { |
|||
final StringBuilder buf = new StringBuilder(); |
|||
int emitted = 0; |
|||
boolean sentinelFound = false; |
|||
} |
|||
|
|||
public static Flux<String> parseFromStream(Flux<String> rawStream, SuggestionCache cache, String chatId) { |
|||
StreamState state = new StreamState(); |
|||
|
|||
return rawStream |
|||
.concatMap(chunk -> Flux.<String>create(sink -> { |
|||
if (state.sentinelFound) { |
|||
sink.complete(); |
|||
return; |
|||
} |
|||
state.buf.append(chunk); |
|||
|
|||
int maxSentinelLen = Math.max(SENTINEL.length(), LOOSE_SENTINEL.length()); |
|||
int safeEnd = Math.max(0, state.buf.length() - maxSentinelLen); |
|||
if (safeEnd <= state.emitted) { |
|||
sink.complete(); |
|||
return; |
|||
} |
|||
|
|||
String safeRegion = state.buf.substring(state.emitted, safeEnd); |
|||
int sentinelIdx = findSentinelIndex(safeRegion); |
|||
|
|||
if (sentinelIdx >= 0) { |
|||
state.sentinelFound = true; |
|||
String beforeSentinel = safeRegion.substring(0, sentinelIdx); |
|||
if (!beforeSentinel.isEmpty()) { |
|||
sink.next(beforeSentinel); |
|||
} |
|||
// 确定实际匹配到的哨兵长度 |
|||
int matchedLen = safeRegion.startsWith(SENTINEL, sentinelIdx) |
|||
? SENTINEL.length() |
|||
: LOOSE_SENTINEL.length(); |
|||
// 哨兵在安全区域内的位置 sentinelIdx，相对全缓冲区即 emitted + sentinelIdx |
|||
// emitted 推进到哨兵结束后，后续内容进 suggestions 解析 |
|||
state.emitted = state.emitted + sentinelIdx + matchedLen; |
|||
log.debug("流式路径检测到哨兵: chatId={}, pos={}, matchedLen={}", chatId, state.emitted, matchedLen); |
|||
} else { |
|||
if (!safeRegion.isEmpty()) { |
|||
sink.next(safeRegion); |
|||
} |
|||
state.emitted = safeEnd; |
|||
} |
|||
sink.complete(); |
|||
})) |
|||
// 流结束后发出安全缓冲区内未发出的残留内容（无哨兵场景） |
|||
.concatWith(Flux.defer(() -> { |
|||
if (!state.sentinelFound && state.emitted < state.buf.length()) { |
|||
String residual = state.buf.substring(state.emitted); |
|||
if (!residual.isEmpty()) { |
|||
return Flux.just(residual); |
|||
} |
|||
} |
|||
return Flux.empty(); |
|||
})) |
|||
.doOnComplete(() -> { |
|||
if (state.sentinelFound) { |
|||
String suggestionsPart = state.buf.length() > state.emitted |
|||
? state.buf.substring(state.emitted) : ""; |
|||
List<String> suggestions = parseSuggestionsJson(suggestionsPart.strip()); |
|||
if (!suggestions.isEmpty()) { |
|||
cache.put(chatId, suggestions); |
|||
log.info("流式 suggestions 解析成功: chatId={}, count={}", chatId, suggestions.size()); |
|||
} else { |
|||
log.debug("流式 suggestions 解析为空: chatId={}", chatId); |
|||
} |
|||
} |
|||
}); |
|||
} |
|||
|
|||
/** |
|||
* 容错解析 suggestions JSON 数组。 |
|||
* 先尝试标准 JSON 解析，失败则按行切分取前 3 条非空行。 |
|||
*/ |
|||
static List<String> parseSuggestionsJson(String jsonPart) { |
|||
if (jsonPart == null || jsonPart.isBlank()) { |
|||
return Collections.emptyList(); |
|||
} |
|||
|
|||
// 1. 尝试标准 JSON 解析 |
|||
String trimmed = jsonPart.strip(); |
|||
// 去掉可能的 markdown 代码块包裹 |
|||
trimmed = trimCodeBlock(trimmed); |
|||
|
|||
try { |
|||
List<String> parsed = objectMapper.readValue(trimmed, new TypeReference<List<String>>() {}); |
|||
List<String> result = new ArrayList<>(); |
|||
for (String s : parsed) { |
|||
if (s != null && !s.isBlank()) { |
|||
result.add(s.strip()); |
|||
} |
|||
} |
|||
if (!result.isEmpty()) { |
|||
return result.size() <= 3 ? result : result.subList(0, 3); |
|||
} |
|||
} catch (Exception e) { |
|||
log.debug("标准 JSON 解析 suggestions 失败，尝试按行切分: {}", e.getMessage()); |
|||
} |
|||
|
|||
// 2. 降级：按行切分，取前 3 条 |
|||
return parseByLines(trimmed); |
|||
} |
|||
|
|||
/** |
|||
* 降级解析：按行切分，去掉序号前缀后取前 3 条非空行。 |
|||
*/ |
|||
private static List<String> parseByLines(String text) { |
|||
List<String> lines = text.lines() |
|||
.map(String::strip) |
|||
.map(SuggestionResponseParser::stripNumberPrefix) |
|||
.map(SuggestionResponseParser::stripQuotes) |
|||
.filter(s -> !s.isBlank()) |
|||
.collect(Collectors.toList()); |
|||
|
|||
if (lines.isEmpty()) { |
|||
return Collections.emptyList(); |
|||
} |
|||
return lines.size() <= 3 ? lines : lines.subList(0, 3); |
|||
} |
|||
|
|||
/** 去掉可能的 markdown 代码块包裹（支持 3 个及以上反引号） */ |
|||
private static String trimCodeBlock(String s) { |
|||
if (s.startsWith("```") && s.endsWith("```")) { |
|||
// 计算开头的反引号数量 |
|||
int openCount = 0; |
|||
while (openCount < s.length() && s.charAt(openCount) == '`') openCount++; |
|||
if (openCount >= 3 && s.endsWith("`".repeat(openCount))) { |
|||
String inner = s.substring(openCount, s.length() - openCount).strip(); |
|||
if (inner.startsWith("json")) { |
|||
inner = inner.substring(4).strip(); |
|||
} else if (inner.startsWith("JSON")) { |
|||
inner = inner.substring(4).strip(); |
|||
} |
|||
return inner; |
|||
} |
|||
} |
|||
return s; |
|||
} |
|||
|
|||
/** 去掉行首序号，如 "1." "2." "3." "1、" "2、" "3、" */ |
|||
private static String stripNumberPrefix(String s) { |
|||
if (s.length() >= 2 && Character.isDigit(s.charAt(0)) && |
|||
(s.charAt(1) == '.' || s.charAt(1) == '、' || s.charAt(1) == ')')) { |
|||
return s.substring(2).strip(); |
|||
} |
|||
return s; |
|||
} |
|||
|
|||
/** 去掉首尾引号 */ |
|||
private static String stripQuotes(String s) { |
|||
if (s.length() >= 2) { |
|||
char first = s.charAt(0); |
|||
char last = s.charAt(s.length() - 1); |
|||
if ((first == '"' && last == '"') || (first == '\'' && last == '\'') || |
|||
(first == '“' && last == '”')) { // 中文引号 " " |
|||
return s.substring(1, s.length() - 1).strip(); |
|||
} |
|||
} |
|||
return s; |
|||
} |
|||
|
|||
/** |
|||
* 解析结果值对象。 |
|||
* |
|||
* @param answer LLM 回答正文（不含哨兵及之后的 suggestions） |
|||
* @param suggestions 建议问题列表（0～3 条） |
|||
*/ |
|||
public record ParsedResponse(String answer, List<String> suggestions) { |
|||
} |
|||
} |
|||
@ -1,85 +0,0 @@ |
|||
package com.wok.supportbot.chatmemory; |
|||
|
|||
import com.esotericsoftware.kryo.Kryo; |
|||
import com.esotericsoftware.kryo.io.Input; |
|||
import com.esotericsoftware.kryo.io.Output; |
|||
import org.objenesis.strategy.StdInstantiatorStrategy; |
|||
import org.springframework.ai.chat.memory.ChatMemory; |
|||
import org.springframework.ai.chat.messages.Message; |
|||
|
|||
import java.io.File; |
|||
import java.io.FileInputStream; |
|||
import java.io.FileOutputStream; |
|||
import java.io.IOException; |
|||
import java.util.ArrayList; |
|||
import java.util.List; |
|||
|
|||
/** |
|||
* 基于文件持久化的对话记忆 |
|||
*/ |
|||
public class FileBasedChatMemory implements ChatMemory { |
|||
|
|||
private final String BASE_DIR; |
|||
private static final Kryo kryo = new Kryo(); |
|||
|
|||
static { |
|||
kryo.setRegistrationRequired(false); |
|||
// 设置实例化策略 |
|||
kryo.setInstantiatorStrategy(new StdInstantiatorStrategy()); |
|||
} |
|||
|
|||
// 构造对象时，指定文件保存目录 |
|||
public FileBasedChatMemory(String dir) { |
|||
this.BASE_DIR = dir; |
|||
File baseDir = new File(dir); |
|||
if (!baseDir.exists()) { |
|||
baseDir.mkdirs(); |
|||
} |
|||
} |
|||
|
|||
@Override |
|||
public void add(String conversationId, List<Message> messages) { |
|||
List<Message> conversationMessages = getOrCreateConversation(conversationId); |
|||
conversationMessages.addAll(messages); |
|||
saveConversation(conversationId, conversationMessages); |
|||
} |
|||
|
|||
@Override |
|||
public List<Message> get(String conversationId) { |
|||
return getOrCreateConversation(conversationId); |
|||
} |
|||
|
|||
@Override |
|||
public void clear(String conversationId) { |
|||
File file = getConversationFile(conversationId); |
|||
if (file.exists()) { |
|||
file.delete(); |
|||
} |
|||
} |
|||
|
|||
private List<Message> getOrCreateConversation(String conversationId) { |
|||
File file = getConversationFile(conversationId); |
|||
List<Message> messages = new ArrayList<>(); |
|||
if (file.exists()) { |
|||
try (Input input = new Input(new FileInputStream(file))) { |
|||
messages = kryo.readObject(input, ArrayList.class); |
|||
} catch (IOException e) { |
|||
e.printStackTrace(); |
|||
} |
|||
} |
|||
return messages; |
|||
} |
|||
|
|||
private void saveConversation(String conversationId, List<Message> messages) { |
|||
File file = getConversationFile(conversationId); |
|||
try (Output output = new Output(new FileOutputStream(file))) { |
|||
kryo.writeObject(output, messages); |
|||
} catch (IOException e) { |
|||
e.printStackTrace(); |
|||
} |
|||
} |
|||
|
|||
private File getConversationFile(String conversationId) { |
|||
return new File(BASE_DIR, conversationId + ".kryo"); |
|||
} |
|||
} |
|||
@ -0,0 +1,40 @@ |
|||
package com.wok.supportbot.common; |
|||
|
|||
import java.util.Map; |
|||
|
|||
/** |
|||
* 列表排序工具类。 |
|||
* <p> |
|||
* 用于统一处理后台列表接口的排序参数：白名单映射（防 SQL 注入）、sortField 判空回退、sortOrder 二值化。 |
|||
* 避免在每个 Service 里重复书写「白名单 + getOrDefault(null) 判空 + asc/desc」逻辑。 |
|||
*/ |
|||
public final class SortUtils { |
|||
|
|||
private SortUtils() { |
|||
} |
|||
|
|||
/** |
|||
* 解析排序字段对应的数据库列名。 |
|||
* <p> |
|||
* sortField 为 null 或不在白名单内时，回退到 defaultColumn，杜绝将用户输入直接拼入 ORDER BY。 |
|||
* |
|||
* @param sortField 前端传入的 colKey（可为 null） |
|||
* @param whitelist 前端 colKey → 数据库列名（可含表别名）的白名单映射 |
|||
* @param defaultColumn 回退列名（通常为 create_time） |
|||
* @return 安全列名 |
|||
*/ |
|||
public static String resolveColumn(String sortField, Map<String, String> whitelist, String defaultColumn) { |
|||
if (sortField == null) { |
|||
return defaultColumn; |
|||
} |
|||
String column = whitelist.get(sortField); |
|||
return column != null ? column : defaultColumn; |
|||
} |
|||
|
|||
/** |
|||
* 排序方向二值化：仅 asc 视为升序，其余（含 null、desc、其他值）一律降序。 |
|||
*/ |
|||
public static boolean isAscending(String sortOrder) { |
|||
return "asc".equalsIgnoreCase(sortOrder); |
|||
} |
|||
} |
|||
@ -0,0 +1,136 @@ |
|||
package com.wok.supportbot.controller; |
|||
|
|||
import com.wok.supportbot.entity.FolderNode; |
|||
import com.wok.supportbot.entity.KnowledgeFolder; |
|||
import com.wok.supportbot.service.FolderService; |
|||
import org.springframework.beans.factory.annotation.Autowired; |
|||
import org.springframework.http.ResponseEntity; |
|||
import org.springframework.security.access.prepost.PreAuthorize; |
|||
import org.springframework.web.bind.annotation.*; |
|||
|
|||
import java.util.List; |
|||
import java.util.Map; |
|||
|
|||
/** |
|||
* 知识库文档目录控制器 |
|||
* 提供目录的树形查询、扁平列表、创建、重命名、删除等功能 |
|||
*/ |
|||
@RestController |
|||
public class FolderController { |
|||
|
|||
@Autowired |
|||
private FolderService folderService; |
|||
|
|||
/** |
|||
* 获取目录树 |
|||
*/ |
|||
@GetMapping("/folder/tree") |
|||
@PreAuthorize("hasAnyRole('admin','kb_operator')") |
|||
public ResponseEntity<Map<String, Object>> getFolderTree(@RequestParam(required = false) Long categoryId) { |
|||
try { |
|||
List<FolderNode> tree = folderService.getFolderTree(categoryId); |
|||
return ResponseEntity.ok(Map.of( |
|||
"success", true, |
|||
"data", tree |
|||
)); |
|||
} catch (Exception e) { |
|||
return ResponseEntity.status(500).body(Map.of( |
|||
"success", false, |
|||
"message", "获取目录树失败：" + e.getMessage() |
|||
)); |
|||
} |
|||
} |
|||
|
|||
/** |
|||
* 获取目录扁平列表 |
|||
*/ |
|||
@GetMapping("/folder/list") |
|||
@PreAuthorize("hasAnyRole('admin','kb_operator')") |
|||
public ResponseEntity<Map<String, Object>> listFolders(@RequestParam(required = false) Long categoryId) { |
|||
try { |
|||
List<KnowledgeFolder> list = folderService.listFolders(categoryId); |
|||
return ResponseEntity.ok(Map.of( |
|||
"success", true, |
|||
"data", list |
|||
)); |
|||
} catch (Exception e) { |
|||
return ResponseEntity.status(500).body(Map.of( |
|||
"success", false, |
|||
"message", "获取目录列表失败：" + e.getMessage() |
|||
)); |
|||
} |
|||
} |
|||
|
|||
/** |
|||
* 创建目录 |
|||
*/ |
|||
@PostMapping("/folder") |
|||
@PreAuthorize("hasAnyRole('admin','kb_operator')") |
|||
public ResponseEntity<Map<String, Object>> createFolder(@RequestBody Map<String, Object> body) { |
|||
try { |
|||
String name = (String) body.get("name"); |
|||
Long categoryId = body.get("categoryId") != null ? Long.valueOf(body.get("categoryId").toString()) : null; |
|||
Long parentId = body.get("parentId") != null ? Long.valueOf(body.get("parentId").toString()) : null; |
|||
Integer sortOrder = body.get("sortOrder") != null ? Integer.valueOf(body.get("sortOrder").toString()) : null; |
|||
|
|||
KnowledgeFolder folder = folderService.createFolder(name, categoryId, parentId, sortOrder); |
|||
return ResponseEntity.ok(Map.of( |
|||
"success", true, |
|||
"message", "目录创建成功", |
|||
"data", folder |
|||
)); |
|||
} catch (Exception e) { |
|||
return ResponseEntity.status(500).body(Map.of( |
|||
"success", false, |
|||
"message", "创建目录失败：" + e.getMessage() |
|||
)); |
|||
} |
|||
} |
|||
|
|||
/** |
|||
* 重命名目录 / 调整排序 |
|||
*/ |
|||
@PutMapping("/folder/{id}") |
|||
@PreAuthorize("hasAnyRole('admin','kb_operator')") |
|||
public ResponseEntity<Map<String, Object>> renameFolder( |
|||
@PathVariable Long id, |
|||
@RequestBody Map<String, Object> body) { |
|||
try { |
|||
String name = (String) body.get("name"); |
|||
Integer sortOrder = body.get("sortOrder") != null ? Integer.valueOf(body.get("sortOrder").toString()) : null; |
|||
|
|||
KnowledgeFolder folder = folderService.renameFolder(id, name, sortOrder); |
|||
return ResponseEntity.ok(Map.of( |
|||
"success", true, |
|||
"message", "目录更新成功", |
|||
"data", folder |
|||
)); |
|||
} catch (Exception e) { |
|||
return ResponseEntity.status(500).body(Map.of( |
|||
"success", false, |
|||
"message", "更新目录失败：" + e.getMessage() |
|||
)); |
|||
} |
|||
} |
|||
|
|||
/** |
|||
* 删除目录（级联删除子孙目录，并将其下文档移到分类根） |
|||
*/ |
|||
@DeleteMapping("/folder/{id}") |
|||
@PreAuthorize("hasAnyRole('admin','kb_operator')") |
|||
public ResponseEntity<Map<String, Object>> deleteFolder(@PathVariable Long id) { |
|||
try { |
|||
Map<String, Object> result = folderService.deleteFolder(id); |
|||
return ResponseEntity.ok(Map.of( |
|||
"success", true, |
|||
"message", "目录删除成功", |
|||
"data", result |
|||
)); |
|||
} catch (Exception e) { |
|||
return ResponseEntity.status(500).body(Map.of( |
|||
"success", false, |
|||
"message", "删除目录失败：" + e.getMessage() |
|||
)); |
|||
} |
|||
} |
|||
} |
|||
@ -0,0 +1,12 @@ |
|||
package com.wok.supportbot.dao; |
|||
|
|||
import com.baomidou.mybatisplus.core.mapper.BaseMapper; |
|||
import com.wok.supportbot.entity.KnowledgeFolder; |
|||
import org.apache.ibatis.annotations.Mapper; |
|||
|
|||
/** |
|||
* 知识库文档目录 Mapper |
|||
*/ |
|||
@Mapper |
|||
public interface KnowledgeFolderMapper extends BaseMapper<KnowledgeFolder> { |
|||
} |
|||
@ -1,73 +0,0 @@ |
|||
package com.wok.supportbot.document.extract; |
|||
|
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.apache.tika.Tika; |
|||
import org.apache.tika.exception.TikaException; |
|||
import org.springframework.ai.document.Document; |
|||
import org.springframework.core.io.Resource; |
|||
import org.springframework.core.io.FileSystemResource; |
|||
import org.springframework.stereotype.Component; |
|||
import org.springframework.web.multipart.MultipartFile; |
|||
|
|||
import java.io.File; |
|||
import java.io.IOException; |
|||
import java.util.Collections; |
|||
import java.util.List; |
|||
import java.util.UUID; |
|||
|
|||
@Component |
|||
@Slf4j |
|||
public class TikaDocumentReader { |
|||
|
|||
/** |
|||
* 从已保存的文件解析内容 |
|||
*/ |
|||
public List<Document> readFromFile(File file) { |
|||
try { |
|||
Tika tika = new Tika(); |
|||
String text = tika.parseToString(new java.io.FileInputStream(file)); |
|||
|
|||
Document doc = Document.builder() |
|||
.id(UUID.randomUUID().toString()) |
|||
.text(text) |
|||
.build(); |
|||
|
|||
return Collections.singletonList(doc); |
|||
|
|||
} catch (IOException | TikaException e) { |
|||
log.error("Tika 文件解析失败", e); |
|||
throw new RuntimeException("Tika 文件解析失败", e); |
|||
} |
|||
} |
|||
|
|||
/** |
|||
* 从 MultipartFile 解析内容（兼容旧接口） |
|||
*/ |
|||
public List<Document> read(MultipartFile file) { |
|||
try { |
|||
Tika tika = new Tika(); |
|||
String text = tika.parseToString(file.getInputStream()); |
|||
|
|||
Document doc = Document.builder() |
|||
.id(UUID.randomUUID().toString()) |
|||
.text(text) |
|||
.build(); |
|||
|
|||
return Collections.singletonList(doc); |
|||
|
|||
} catch (IOException | TikaException e) { |
|||
log.error("Tika 文件解析失败", e); |
|||
throw new RuntimeException("Tika 文件解析失败", e); |
|||
} |
|||
} |
|||
|
|||
/** |
|||
* 获取文件扩展名（含点号），如 ".pptx"。无扩展名时返回 ".tmp" |
|||
*/ |
|||
private String getExtension(String filename) { |
|||
if (filename == null || !filename.contains(".")) { |
|||
return ".tmp"; |
|||
} |
|||
return filename.substring(filename.lastIndexOf(".")); |
|||
} |
|||
} |
|||
@ -1,30 +0,0 @@ |
|||
package com.wok.supportbot.document.transform; |
|||
|
|||
import com.wok.supportbot.config.ChatModelFactory; |
|||
import jakarta.annotation.Resource; |
|||
import org.springframework.ai.document.Document; |
|||
import org.springframework.ai.model.transformer.KeywordMetadataEnricher; |
|||
import org.springframework.stereotype.Component; |
|||
|
|||
import java.util.List; |
|||
|
|||
/** |
|||
* 基于 AI 的文档元信息增强器（为文档补充元信息） |
|||
* 通过 ChatModelFactory 获取 ChatModel，支持多提供商动态切换 |
|||
*/ |
|||
@Component |
|||
public class MyKeywordEnricher { |
|||
|
|||
@Resource |
|||
private ChatModelFactory chatModelFactory; |
|||
|
|||
/** |
|||
* 使用 AI 提取关键词并添加到元数据 |
|||
*/ |
|||
public List<Document> enrichDocuments(List<Document> documents) { |
|||
KeywordMetadataEnricher enricher = new KeywordMetadataEnricher.Builder(chatModelFactory.getChatModel("CHAT")) |
|||
.keywordCount(5) |
|||
.build(); |
|||
return enricher.apply(documents); |
|||
} |
|||
} |
|||
@ -1,67 +0,0 @@ |
|||
package com.wok.supportbot.document.transform; |
|||
|
|||
import com.wok.supportbot.config.ChunkConfig; |
|||
import org.springframework.ai.document.Document; |
|||
import org.springframework.ai.transformer.splitter.TokenTextSplitter; |
|||
import org.springframework.beans.factory.annotation.Autowired; |
|||
import org.springframework.stereotype.Component; |
|||
|
|||
import java.util.List; |
|||
|
|||
/** |
|||
* 自定义基于 Token 的切词器 |
|||
* 支持通过 ChunkConfig 动态调整分块参数 |
|||
*/ |
|||
@Component |
|||
public class MyTokenTextSplitter { |
|||
|
|||
@Autowired |
|||
private ChunkConfig chunkConfig; |
|||
|
|||
/** |
|||
* 使用全局配置参数创建分割器 |
|||
*/ |
|||
public List<Document> splitDocuments(List<Document> documents) { |
|||
TokenTextSplitter splitter = new TokenTextSplitter( |
|||
chunkConfig.getChunkSize(), |
|||
chunkConfig.getOverlap(), |
|||
chunkConfig.getMinChunkSizeChars(), |
|||
chunkConfig.getMaxNumChunks(), |
|||
chunkConfig.isKeepSeparator() |
|||
); |
|||
return splitter.apply(documents); |
|||
} |
|||
|
|||
/** |
|||
* 使用自定义参数创建分割器（覆盖全局配置） |
|||
* |
|||
* @param documents 文档列表 |
|||
* @param chunkSize 分块大小 |
|||
* @param overlap 重叠大小 |
|||
*/ |
|||
public List<Document> splitDocuments(List<Document> documents, Integer chunkSize, Integer overlap) { |
|||
int cs = chunkSize != null ? chunkSize : chunkConfig.getChunkSize(); |
|||
int ol = overlap != null ? overlap : chunkConfig.getOverlap(); |
|||
TokenTextSplitter splitter = new TokenTextSplitter( |
|||
cs, ol, |
|||
chunkConfig.getMinChunkSizeChars(), |
|||
chunkConfig.getMaxNumChunks(), |
|||
chunkConfig.isKeepSeparator() |
|||
); |
|||
return splitter.apply(documents); |
|||
} |
|||
|
|||
/** |
|||
* 使用自定义参数创建分割器（全参数覆盖） |
|||
*/ |
|||
public List<Document> splitCustomized(List<Document> documents) { |
|||
TokenTextSplitter splitter = new TokenTextSplitter( |
|||
chunkConfig.getChunkSize(), |
|||
chunkConfig.getOverlap(), |
|||
chunkConfig.getMinChunkSizeChars(), |
|||
chunkConfig.getMaxNumChunks(), |
|||
chunkConfig.isKeepSeparator() |
|||
); |
|||
return splitter.apply(documents); |
|||
} |
|||
} |
|||
@ -0,0 +1,252 @@ |
|||
package com.wok.supportbot.document.transform; |
|||
|
|||
import com.knuddels.jtokkit.Encodings; |
|||
import com.knuddels.jtokkit.api.Encoding; |
|||
import com.knuddels.jtokkit.api.EncodingType; |
|||
import com.knuddels.jtokkit.api.IntArrayList; |
|||
import org.slf4j.Logger; |
|||
import org.slf4j.LoggerFactory; |
|||
import org.springframework.ai.transformer.splitter.TextSplitter; |
|||
import org.springframework.util.Assert; |
|||
|
|||
import java.util.ArrayList; |
|||
import java.util.List; |
|||
|
|||
/** |
|||
* 带重叠（overlap）的 Token 分块器。 |
|||
* |
|||
* <p>Spring AI 的 {@link org.springframework.ai.transformer.splitter.TokenTextSplitter} |
|||
* 不支持 overlap —— 它的 5 参构造器与 Builder 均无该形参（社区 PR #4054 已标记不向 1.x 回迁）。 |
|||
* 本类直接继承标准基类 {@link TextSplitter}，复刻 TokenTextSplitter 的全部切分语义 |
|||
* （标点截断、最小分块长度、分块数上限、分隔符处理），仅把「前进步长」由 chunkSize |
|||
* 改为 {@code chunkSize - overlap}，从而让重叠真正生效。 |
|||
* |
|||
* <p>元数据继承（{@code parent_document_id} / {@code chunk_index} / {@code total_chunks}） |
|||
* 由 {@link TextSplitter} 基类统一处理，本类只负责文本切分。 |
|||
* |
|||
* <p>Token 编码与 TokenTextSplitter 保持一致：jtokkit + CL100K_BASE。 |
|||
*/ |
|||
public class OverlapTokenTextSplitter extends TextSplitter { |
|||
|
|||
private static final Logger log = LoggerFactory.getLogger(OverlapTokenTextSplitter.class); |
|||
|
|||
/** 默认分块大小（token 数） */ |
|||
private static final int DEFAULT_CHUNK_SIZE = 800; |
|||
|
|||
/** 默认重叠 token 数（0 表示不重叠，等价于 TokenTextSplitter 行为） */ |
|||
private static final int DEFAULT_OVERLAP = 0; |
|||
|
|||
/** 默认最小分块字符数 */ |
|||
private static final int MIN_CHUNK_SIZE_CHARS = 350; |
|||
|
|||
/** 默认丢弃阈值：长度不超过该值的分块不入库 */ |
|||
private static final int MIN_CHUNK_LENGTH_TO_EMBED = 5; |
|||
|
|||
/** 默认单篇文档最大分块数 */ |
|||
private static final int MAX_NUM_CHUNKS = 10000; |
|||
|
|||
/** 默认保留分隔符 */ |
|||
private static final boolean KEEP_SEPARATOR = true; |
|||
|
|||
private final Encoding encoding = Encodings.newLazyEncodingRegistry().getEncoding(EncodingType.CL100K_BASE); |
|||
|
|||
/** 目标分块大小（token 数，不含重叠部分） */ |
|||
private final int chunkSize; |
|||
|
|||
/** 相邻分块的重叠 token 数 */ |
|||
private final int overlap; |
|||
|
|||
/** 最小分块字符数：仅当剩余 token 数超过 chunkSize 时才按标点截断，且截断点须大于该值 */ |
|||
private final int minChunkSizeChars; |
|||
|
|||
/** 丢弃长度不超过该值的分块 */ |
|||
private final int minChunkLengthToEmbed; |
|||
|
|||
/** 单篇文档最大分块数 */ |
|||
private final int maxNumChunks; |
|||
|
|||
/** 是否保留分隔符（false 时把换行替换为空格） */ |
|||
private final boolean keepSeparator; |
|||
|
|||
private OverlapTokenTextSplitter(int chunkSize, int overlap, int minChunkSizeChars, |
|||
int minChunkLengthToEmbed, int maxNumChunks, boolean keepSeparator) { |
|||
Assert.isTrue(chunkSize > 0, "chunkSize 必须大于 0"); |
|||
this.chunkSize = chunkSize; |
|||
// 重叠必须严格小于分块大小，否则前进步长会 <= 0 导致死循环；此处收敛到合法区间而非直接报错 |
|||
int clamped = Math.max(0, Math.min(overlap, chunkSize - 1)); |
|||
if (clamped != overlap) { |
|||
log.warn("overlap({}) 超出合法区间 [0, chunkSize-1={}]，已收敛为 {}；" |
|||
+ "该配置会使前进步长退化为 1 个 token，产生大量分块，请检查 knowledge.chunk.overlap", |
|||
overlap, chunkSize - 1, clamped); |
|||
} |
|||
this.overlap = clamped; |
|||
this.minChunkSizeChars = minChunkSizeChars; |
|||
this.minChunkLengthToEmbed = minChunkLengthToEmbed; |
|||
this.maxNumChunks = maxNumChunks; |
|||
this.keepSeparator = keepSeparator; |
|||
} |
|||
|
|||
public static Builder builder() { |
|||
return new Builder(); |
|||
} |
|||
|
|||
/** |
|||
* 标点截断允许的最小前进步长（token 数）。 |
|||
* <p> |
|||
* 防止「标点截断」与「overlap」叠加后把前进步长压到 1 个 token 导致分块数爆炸： |
|||
* 截断后必须仍能前进这么多 token，否则放弃本次截断（宁可切断句子也要保证分块数量可控）。 |
|||
* <p> |
|||
* overlap=0 时返回 1，即允许任意截断，与标准 {@code TokenTextSplitter} 行为完全一致。 |
|||
*/ |
|||
private int minAdvance() { |
|||
if (this.overlap == 0) { |
|||
return 1; |
|||
} |
|||
return Math.max(1, (this.chunkSize - this.overlap) / 2); |
|||
} |
|||
|
|||
@Override |
|||
protected List<String> splitText(String text) { |
|||
if (text == null || text.trim().isEmpty()) { |
|||
return new ArrayList<>(); |
|||
} |
|||
|
|||
List<Integer> tokens = getEncodedTokens(text); |
|||
List<String> chunks = new ArrayList<>(); |
|||
int numChunks = 0; |
|||
|
|||
while (!tokens.isEmpty() && numChunks < this.maxNumChunks) { |
|||
int windowSize = Math.min(this.chunkSize, tokens.size()); |
|||
boolean lastWindow = windowSize >= tokens.size(); |
|||
String chunkText = decodeTokens(tokens.subList(0, windowSize)); |
|||
|
|||
// 空白块直接跳过（不产出，也不做重叠回退） |
|||
if (chunkText.trim().isEmpty()) { |
|||
tokens = tokens.subList(windowSize, tokens.size()); |
|||
continue; |
|||
} |
|||
|
|||
// 仅当剩余 token 数超过 chunkSize 时才做标点截断，避免小文本被无谓切分 |
|||
if (tokens.size() > this.chunkSize) { |
|||
int lastPunctuation = Math.max(chunkText.lastIndexOf('.'), |
|||
Math.max(chunkText.lastIndexOf('?'), |
|||
Math.max(chunkText.lastIndexOf('!'), chunkText.lastIndexOf('\n')))); |
|||
if (lastPunctuation != -1 && lastPunctuation > this.minChunkSizeChars) { |
|||
String candidate = chunkText.substring(0, lastPunctuation + 1); |
|||
// 前进步长 = 截断后消耗的 token 数 - overlap。截断会缩短消耗量， |
|||
// 若不加约束，截断点靠前时步长会被压到 1 个 token，分块数成倍膨胀 |
|||
// （实测 chunkSize=60/overlap=30 时可达 10 倍）。此处要求截断后仍能前进至少 minAdvance()。 |
|||
if (getEncodedTokens(candidate).size() - this.overlap >= minAdvance()) { |
|||
chunkText = candidate; |
|||
} |
|||
} |
|||
} |
|||
|
|||
String chunkTextToAppend = this.keepSeparator |
|||
? chunkText.trim() |
|||
: chunkText.replace(System.lineSeparator(), " ").trim(); |
|||
if (chunkTextToAppend.length() > this.minChunkLengthToEmbed) { |
|||
chunks.add(chunkTextToAppend); |
|||
} |
|||
numChunks++; |
|||
|
|||
// 本窗口已覆盖全部剩余 token，无需再产出重叠块 |
|||
if (lastWindow) { |
|||
tokens = new ArrayList<>(); |
|||
break; |
|||
} |
|||
|
|||
// 本块实际消耗的 token 数（可能因标点截断而少于窗口大小） |
|||
int consumed = getEncodedTokens(chunkText).size(); |
|||
// 前进步长 = 消耗量 - 重叠量；至少前进 1 个 token，避免死循环 |
|||
int step = Math.max(1, consumed - this.overlap); |
|||
tokens = tokens.subList(step, tokens.size()); |
|||
} |
|||
|
|||
// 处理剩余 token（与 TokenTextSplitter 保持一致，用于达到 maxNumChunks 上限的场景） |
|||
if (!tokens.isEmpty()) { |
|||
String remainingText = decodeTokens(tokens).replace(System.lineSeparator(), " ").trim(); |
|||
if (remainingText.length() > this.minChunkLengthToEmbed) { |
|||
chunks.add(remainingText); |
|||
} |
|||
} |
|||
|
|||
return chunks; |
|||
} |
|||
|
|||
private List<Integer> getEncodedTokens(String text) { |
|||
Assert.notNull(text, "Text must not be null"); |
|||
return this.encoding.encode(text).boxed(); |
|||
} |
|||
|
|||
private String decodeTokens(List<Integer> tokens) { |
|||
Assert.notNull(tokens, "Tokens must not be null"); |
|||
IntArrayList tokensIntArray = new IntArrayList(tokens.size()); |
|||
tokens.forEach(tokensIntArray::add); |
|||
return this.encoding.decode(tokensIntArray); |
|||
} |
|||
|
|||
/** |
|||
* 分块器构建器 |
|||
*/ |
|||
public static final class Builder { |
|||
|
|||
private int chunkSize = DEFAULT_CHUNK_SIZE; |
|||
|
|||
private int overlap = DEFAULT_OVERLAP; |
|||
|
|||
private int minChunkSizeChars = MIN_CHUNK_SIZE_CHARS; |
|||
|
|||
private int minChunkLengthToEmbed = MIN_CHUNK_LENGTH_TO_EMBED; |
|||
|
|||
private int maxNumChunks = MAX_NUM_CHUNKS; |
|||
|
|||
private boolean keepSeparator = KEEP_SEPARATOR; |
|||
|
|||
private Builder() { |
|||
} |
|||
|
|||
/** 目标分块大小（token 数） */ |
|||
public Builder withChunkSize(int chunkSize) { |
|||
this.chunkSize = chunkSize; |
|||
return this; |
|||
} |
|||
|
|||
/** 相邻分块的重叠 token 数 */ |
|||
public Builder withOverlap(int overlap) { |
|||
this.overlap = overlap; |
|||
return this; |
|||
} |
|||
|
|||
/** 最小分块字符数（标点截断的下限） */ |
|||
public Builder withMinChunkSizeChars(int minChunkSizeChars) { |
|||
this.minChunkSizeChars = minChunkSizeChars; |
|||
return this; |
|||
} |
|||
|
|||
/** 丢弃长度不超过该值的分块 */ |
|||
public Builder withMinChunkLengthToEmbed(int minChunkLengthToEmbed) { |
|||
this.minChunkLengthToEmbed = minChunkLengthToEmbed; |
|||
return this; |
|||
} |
|||
|
|||
/** 单篇文档最大分块数 */ |
|||
public Builder withMaxNumChunks(int maxNumChunks) { |
|||
this.maxNumChunks = maxNumChunks; |
|||
return this; |
|||
} |
|||
|
|||
/** 是否保留分隔符 */ |
|||
public Builder withKeepSeparator(boolean keepSeparator) { |
|||
this.keepSeparator = keepSeparator; |
|||
return this; |
|||
} |
|||
|
|||
public OverlapTokenTextSplitter build() { |
|||
return new OverlapTokenTextSplitter(this.chunkSize, this.overlap, this.minChunkSizeChars, |
|||
this.minChunkLengthToEmbed, this.maxNumChunks, this.keepSeparator); |
|||
} |
|||
|
|||
} |
|||
|
|||
} |
|||
@ -0,0 +1,63 @@ |
|||
package com.wok.supportbot.entity; |
|||
|
|||
import com.fasterxml.jackson.databind.annotation.JsonSerialize; |
|||
import com.fasterxml.jackson.databind.ser.std.ToStringSerializer; |
|||
import lombok.AllArgsConstructor; |
|||
import lombok.Builder; |
|||
import lombok.Data; |
|||
import lombok.NoArgsConstructor; |
|||
|
|||
import java.io.Serial; |
|||
import java.io.Serializable; |
|||
import java.util.List; |
|||
|
|||
/** |
|||
* 目录树节点 - 用于返回树形结构 |
|||
*/ |
|||
@Data |
|||
@Builder |
|||
@AllArgsConstructor |
|||
@NoArgsConstructor |
|||
public class FolderNode implements Serializable { |
|||
|
|||
@Serial |
|||
private static final long serialVersionUID = 1L; |
|||
|
|||
/** |
|||
* 目录ID |
|||
*/ |
|||
@JsonSerialize(using = ToStringSerializer.class) |
|||
private Long id; |
|||
|
|||
/** |
|||
* 目录名称 |
|||
*/ |
|||
private String name; |
|||
|
|||
/** |
|||
* 所属分类ID |
|||
*/ |
|||
@JsonSerialize(using = ToStringSerializer.class) |
|||
private Long categoryId; |
|||
|
|||
/** |
|||
* 父目录ID |
|||
*/ |
|||
@JsonSerialize(using = ToStringSerializer.class) |
|||
private Long parentId; |
|||
|
|||
/** |
|||
* 排序权重 |
|||
*/ |
|||
private Integer sortOrder; |
|||
|
|||
/** |
|||
* 目录内文档数量 |
|||
*/ |
|||
private Integer documentCount; |
|||
|
|||
/** |
|||
* 子目录列表 |
|||
*/ |
|||
private List<FolderNode> children; |
|||
} |
|||
@ -0,0 +1,80 @@ |
|||
package com.wok.supportbot.entity; |
|||
|
|||
import com.baomidou.mybatisplus.annotation.*; |
|||
import com.fasterxml.jackson.databind.annotation.JsonSerialize; |
|||
import com.fasterxml.jackson.databind.ser.std.ToStringSerializer; |
|||
import lombok.AllArgsConstructor; |
|||
import lombok.Builder; |
|||
import lombok.Data; |
|||
import lombok.NoArgsConstructor; |
|||
|
|||
import java.io.Serial; |
|||
import java.io.Serializable; |
|||
import java.util.Date; |
|||
|
|||
/** |
|||
* 知识库文档目录表 - 支持分类下的目录树形结构 |
|||
*/ |
|||
@Data |
|||
@Builder |
|||
@AllArgsConstructor |
|||
@NoArgsConstructor |
|||
@TableName("knowledge_folder") |
|||
public class KnowledgeFolder implements Serializable { |
|||
|
|||
@Serial |
|||
@TableField(exist = false) |
|||
private static final long serialVersionUID = 1L; |
|||
|
|||
/** |
|||
* 目录ID |
|||
*/ |
|||
@TableId(value = "id", type = IdType.ASSIGN_ID) |
|||
@JsonSerialize(using = ToStringSerializer.class) |
|||
private Long id; |
|||
|
|||
/** |
|||
* 所属分类ID |
|||
*/ |
|||
@TableField("category_id") |
|||
@JsonSerialize(using = ToStringSerializer.class) |
|||
private Long categoryId; |
|||
|
|||
/** |
|||
* 父目录ID，0=该分类下的根目录 |
|||
*/ |
|||
@TableField("parent_id") |
|||
@JsonSerialize(using = ToStringSerializer.class) |
|||
private Long parentId; |
|||
|
|||
/** |
|||
* 目录名称 |
|||
*/ |
|||
@TableField("name") |
|||
private String name; |
|||
|
|||
/** |
|||
* 排序权重(越大越靠前) |
|||
*/ |
|||
@TableField("sort_order") |
|||
private Integer sortOrder; |
|||
|
|||
/** |
|||
* 创建时间 |
|||
*/ |
|||
@TableField(value = "create_time", fill = FieldFill.INSERT) |
|||
private Date createTime; |
|||
|
|||
/** |
|||
* 更新时间 |
|||
*/ |
|||
@TableField(value = "update_time", fill = FieldFill.INSERT_UPDATE) |
|||
private Date updateTime; |
|||
|
|||
/** |
|||
* 逻辑删除标志 - false:未删除, true:已删除(逻辑删除) |
|||
*/ |
|||
@TableField("is_delete") |
|||
@TableLogic |
|||
private boolean isDelete; |
|||
} |
|||
Some files were not shown because too many files changed in this diff
Write
Preview
Loading…
Cancel
Save
Reference in new issue