Browse Source

feat(sdk): 支持图片上传与多模态对话

SDK 前端新增图片附件上传与多模态对话能力,输入区新增附件按钮与图片预览,新增 allowImageUpload 配置项;/attachment/upload 鉴权改由 SdkAuthFilter 承担(移除方法级 @PreAuthorize),JwtAuthFilter/SecurityConfig 同步放行
Spring-AI-1.1.2
wanghanlin 3 weeks ago
parent
commit
d5d392aca4
  1. 68
      client/src/api.ts
  2. 120
      client/src/chat.ts
  3. 1
      client/src/config.ts
  4. 28
      client/src/dom.ts
  5. 2
      client/src/i18n.ts
  6. 1
      client/src/index.ts
  7. 20
      client/src/types.ts
  8. 6
      src/main/java/com/wok/supportbot/controller/AttachmentController.java
  9. 4
      src/main/java/com/wok/supportbot/security/JwtAuthFilter.java
  10. 4
      src/main/java/com/wok/supportbot/security/SdkAuthFilter.java
  11. 2
      src/main/java/com/wok/supportbot/security/SecurityConfig.java
  12. 2
      src/main/resources/static/sdk/test.html

68
client/src/api.ts

@ -6,7 +6,7 @@
* userId accountId ID
* chatId ID /ai/sdk/conversation/list
*/
import { ResolvedConfig, ApiResponse, CategoryNode } from './types';
import { ResolvedConfig, ApiResponse, CategoryNode, ImageAttachment } from './types';
import { logger } from './logger';
import { t } from './i18n';
@ -72,13 +72,22 @@ function setIfPresent(params: URLSearchParams, key: string, value: string | numb
// ==================== 对话接口 URL 构建 ====================
/**
* URL encodeURIComponent
*/
function appendImageUrls(params: URLSearchParams, imageUrls?: string[]): void {
if (imageUrls && imageUrls.length > 0) {
params.set('imageUrls', imageUrls.map(u => encodeURIComponent(u)).join(','));
}
}
/**
* URL
* - integrateId roleId
* - userId accountId
* - chatId ID
*/
function buildChatUrl(message: string): string {
function buildChatUrl(message: string, imageUrls?: string[]): string {
const params = new URLSearchParams();
params.set('message', message);
params.set('chatId', currentConfig!.chatId);
@ -87,6 +96,7 @@ function buildChatUrl(message: string): string {
setIfPresent(params, 'roleId', getActiveIntegrateId());
// userId 映射为 accountId
setIfPresent(params, 'accountId', currentConfig!.userId);
appendImageUrls(params, imageUrls);
return buildUrl(`/ai/chat?${params.toString()}`);
}
@ -94,7 +104,7 @@ function buildChatUrl(message: string): string {
/**
* SSE URL
*/
function buildChatSSEUrl(message: string, categoryId?: number, useRag?: boolean): string {
function buildChatSSEUrl(message: string, categoryId?: number, useRag?: boolean, imageUrls?: string[]): string {
const params = new URLSearchParams();
params.set('message', message);
params.set('chatId', currentConfig!.chatId);
@ -108,6 +118,7 @@ function buildChatSSEUrl(message: string, categoryId?: number, useRag?: boolean)
params.set('enableRag', 'true');
params.set('rewriteStrategy', currentConfig!.rewriteStrategy || 'REWRITE');
}
appendImageUrls(params, imageUrls);
return buildUrl(`/ai/chat/stream?${params.toString()}`);
}
@ -161,8 +172,9 @@ async function safeFetch(
Object.assign(headers, options.headers as Record<string, string>);
}
}
// 为受 SdkAuthFilter 保护的路径自动注入 Bearer Token(/ai/** 和 /feedback)
if (currentConfig?.token && (url.includes('/ai/') || url.endsWith('/feedback'))) {
// 为受 SdkAuthFilter 保护的路径自动注入 Bearer Token(/ai/**、/feedback、/attachment/upload)
if (currentConfig?.token &&
(url.includes('/ai/') || url.endsWith('/feedback') || url.includes('/attachment/upload'))) {
headers['Authorization'] = `Bearer ${currentConfig.token}`;
}
@ -223,8 +235,8 @@ function getHttpErrorMessage(status: number): string {
/**
*
*/
export async function chatRequest(message: string): Promise<string> {
const url = buildChatUrl(message);
export async function chatRequest(message: string, imageUrls?: string[]): Promise<string> {
const url = buildChatUrl(message, imageUrls);
logger.lifecycleSend(getActiveIntegrateId(), message.length);
try {
@ -257,11 +269,12 @@ export async function chatSSERequest(
onError: (error: CskError) => void,
categoryId?: number,
useRag?: boolean,
imageUrls?: string[],
signal?: AbortSignal
): Promise<void> {
const url = useRag
? buildChatSSEUrl(message, categoryId, true)
: buildChatSSEUrl(message, categoryId, false);
? buildChatSSEUrl(message, categoryId, true, imageUrls)
: buildChatSSEUrl(message, categoryId, false, imageUrls);
let totalText = '';
/**
@ -449,6 +462,43 @@ export async function chatSSERequest(
}
}
// ==================== 图片上传 ====================
/** 上传超时时间(图片体积较大,放宽到 120s) */
const UPLOAD_TIMEOUT = 120000;
/**
* /attachment/upload访 URL
*/
export async function uploadAttachment(file: File): Promise<ImageAttachment> {
const url = buildUrl('/attachment/upload');
const fd = new FormData();
fd.append('file', file);
const response = await safeFetch(url, { method: 'POST', body: fd }, UPLOAD_TIMEOUT);
if (!response.ok) {
throw new CskError(getHttpErrorMessage(response.status), `http_${response.status}`);
}
const json = await response.json() as ApiResponse<{
url: string;
name?: string;
mimeType?: string;
size?: number;
}>;
if (!json.success || !json.data?.url) {
throw new CskError(json.message || t('error_upload_failed'), 'upload_failed');
}
return {
name: json.data.name || file.name,
url: json.data.url,
mimeType: json.data.mimeType,
size: json.data.size,
};
}
// ==================== P1: 知识库分类 ====================
/**

120
client/src/chat.ts

@ -6,7 +6,7 @@
* userId accountId ID
* chatId /conversation/list
*/
import { ResolvedConfig, ChatMessage, RagSource } from './types';
import { ResolvedConfig, ChatMessage, RagSource, ImageAttachment } from './types';
import {
chatRequest,
chatSSERequest,
@ -26,6 +26,7 @@ import {
getActiveIntegrateId,
CskError,
fetchSuggestions,
uploadAttachment,
} from './api';
import {
renderUserBubble,
@ -83,6 +84,9 @@ let currentCategoryId: number | undefined;
/** 当前是否使用 RAG 对话 */
let useRag = false;
/** 待发送的图片列表(与 t-chat-sender 的 attachmentsProps.items 同步) */
let pendingImages: ImageAttachment[] = [];
/**
*
*/
@ -195,12 +199,20 @@ async function loadHistoryFromBackend(): Promise<void> {
function bindSendEvents(): void {
if (!inputEl) return;
// t-chat-sender:send 事件(点击发送/回车)携带 value;stop 事件(loading 态点击)中断流式
// t-chat-sender:send 事件(点击发送/回车)携带 value 与 attachments;stop 事件(loading 态点击)中断流式
inputEl.addEventListener('send', (e) => {
const detail = (e as CustomEvent).detail as { value?: string } | undefined;
const detail = (e as CustomEvent).detail as {
value?: string;
attachments?: Array<{ name?: string; url?: string; fileType?: string; size?: number }>;
} | undefined;
const value = (detail?.value || '').trim();
if (!value || isSending) return;
handleSend(value);
// 从附件中提取图片(本次仅图片参与多模态对话)
const images: ImageAttachment[] = (detail?.attachments || [])
.filter(a => a.fileType === 'image' && a.url)
.map(a => ({ name: a.name || '', url: a.url as string, size: a.size }));
if ((!value && images.length === 0) || isSending) return;
handleSend(value, images);
});
inputEl.addEventListener('stop', () => {
@ -209,11 +221,61 @@ function bindSendEvents(): void {
}
});
// t-chat-sender:fileSelect 事件(选择文件后触发,携带原始 File[])
inputEl.addEventListener('fileSelect', (e) => {
const files = (e as CustomEvent<File[]>).detail;
if (!files || files.length === 0) return;
handleFileSelect(files);
});
if (clearBtn) {
clearBtn.addEventListener('click', () => handleClear());
}
}
/**
* /attachment/upload
*/
async function handleFileSelect(files: File[]): Promise<void> {
if (!config) return;
for (const file of files) {
try {
const att = await uploadAttachment(file);
pendingImages.push(att);
} catch (err) {
const msg = err instanceof CskError ? err.message : t('error_upload_failed');
if (messagesContainer) renderErrorBubble(messagesContainer, msg, now());
logger.error('图片上传失败', err);
}
}
syncSenderAttachments();
}
/**
* pendingImages t-chat-sender attachmentsProps.items
* TDesign Chat
*/
function syncSenderAttachments(): void {
if (!inputEl) return;
const items = pendingImages.map(img => ({
name: img.name,
url: img.url,
fileType: 'image' as const,
size: img.size,
}));
(inputEl as unknown as {
attachmentsProps: { items: typeof items; overflow: string };
}).attachmentsProps = { items, overflow: 'scrollX' };
}
/** 清空待发送图片(发送成功后、清空会话、切换角色/会话时调用) */
function clearPendingImages(): void {
pendingImages = [];
syncSenderAttachments();
}
/** 绑定滚动监听:判断是否在底部,控制新消息提示按钮 */
function bindScrollEvents(): void {
if (!messagesContainer) return;
@ -404,16 +466,20 @@ function updateEmptyState(): void {
}
/** 处理发送消息(text 来自 t-chat-sender 的 send 事件,或快捷问题/重试的显式调用) */
async function handleSend(text?: string): Promise<void> {
async function handleSend(text?: string, images?: ImageAttachment[]): Promise<void> {
if (!config || isSending) return;
const input = (text ?? '').trim();
if (input === '') return;
const atts = images && images.length > 0 ? images : [...pendingImages];
if (input === '' && atts.length === 0) return;
// 清空待发送图片(快照已保存到 atts,发送失败时恢复)
clearPendingImages();
// 1. 渲染用户气泡
const userTimestamp = now();
const userMsg: ChatMessage = { id: uuid(), role: 'user', content: input, timestamp: userTimestamp };
if (messagesContainer) renderUserBubble(messagesContainer, input, userTimestamp);
const userMsg: ChatMessage = { id: uuid(), role: 'user', content: input, timestamp: userTimestamp, images: atts };
if (messagesContainer) renderUserBubble(messagesContainer, input, userTimestamp, atts);
messages.push(userMsg);
updateEmptyState();
@ -421,15 +487,21 @@ async function handleSend(text?: string): Promise<void> {
if (messagesContainer) smartScrollToBottom();
// 2. 生成 AI 回复
await produceAIReply(input);
const ok = await produceAIReply(input, atts);
// 发送失败时恢复待发送图片,方便用户重试
if (!ok && atts.length > 0) {
pendingImages = atts;
syncSenderAttachments();
}
}
/**
* AI + +
*
* @returns false
*/
async function produceAIReply(userText: string): Promise<void> {
if (!config || !messagesContainer) return;
async function produceAIReply(userText: string, images?: ImageAttachment[]): Promise<boolean> {
if (!config || !messagesContainer) return false;
isSending = true;
setSendButtonMode('stop');
@ -439,6 +511,9 @@ async function produceAIReply(userText: string): Promise<void> {
await initChatId();
}
// 提取图片 URL,仅图片参与多模态对话
const imageUrls = (images || []).map(img => img.url);
const aiTimestamp = now();
// RAG 启用条件:由 enableRag 控制
const shouldUseRag = useRag;
@ -452,9 +527,9 @@ async function produceAIReply(userText: string): Promise<void> {
try {
if (config.streaming) {
aiContent = await sendStreamMessage(userText, aiTimestamp, shouldUseRag, aiMsgId);
aiContent = await sendStreamMessage(userText, aiTimestamp, shouldUseRag, aiMsgId, imageUrls);
} else {
aiContent = await chatRequest(userText);
aiContent = await chatRequest(userText, imageUrls);
if (hideLoadingFn) hideLoadingFn();
if (messagesContainer) {
renderAIBubble(messagesContainer, aiContent, aiTimestamp, aiMsgId);
@ -484,6 +559,7 @@ async function produceAIReply(userText: string): Promise<void> {
// 发送成功后清除离线横幅(网络已恢复)
hideOfflineBanner();
return true;
} catch (err) {
if (hideLoadingFn) hideLoadingFn();
@ -492,6 +568,7 @@ async function produceAIReply(userText: string): Promise<void> {
renderErrorBubble(messagesContainer, errMsg, now());
}
logger.error(`发送失败 integrateId=${config.integrateId}`, err);
return false;
} finally {
isSending = false;
abortController = null;
@ -500,7 +577,7 @@ async function produceAIReply(userText: string): Promise<void> {
}
/** 流式发送消息 */
async function sendStreamMessage(text: string, aiTimestamp: number, shouldUseRag: boolean, aiMsgId: string): Promise<string> {
async function sendStreamMessage(text: string, aiTimestamp: number, shouldUseRag: boolean, aiMsgId: string, imageUrls?: string[]): Promise<string> {
// 创建中断控制器,供"停止生成"使用
abortController = new AbortController();
const signal = abortController.signal;
@ -535,7 +612,7 @@ async function sendStreamMessage(text: string, aiTimestamp: number, shouldUseRag
// 无流内容降级为同步请求(须在 wrapperEl/bubbleEl 判断之外:
// 二者仅在 onChunk 收到首个 token 时才赋值,否则此分支不可达)
if (!streamStarted && accumulated === '') {
chatRequest(text).then(resolve).catch(reject);
chatRequest(text, imageUrls).then(resolve).catch(reject);
return;
}
if (wrapperEl && bubbleEl) {
@ -563,6 +640,7 @@ async function sendStreamMessage(text: string, aiTimestamp: number, shouldUseRag
},
currentCategoryId,
shouldUseRag,
imageUrls,
signal
);
});
@ -597,7 +675,8 @@ export async function retryFromMessage(msgId: string): Promise<void> {
}
if (userIndex < 0) return;
const userText = messages[userIndex].content;
if (!userText) return;
const userImages = messages[userIndex].images || [];
if (!userText && userImages.length === 0) return;
// 计算 userTurn(1-based,仅统计 USER 消息):该轮提问是第几条用户消息
let userTurn = 0;
@ -623,7 +702,7 @@ export async function retryFromMessage(msgId: string): Promise<void> {
renderHistory();
// 4. 重新发送该轮提问(重新渲染用户气泡 + 生成新回复)
await handleSend(userText);
await handleSend(userText, userImages);
} finally {
isRetrying = false;
}
@ -694,7 +773,7 @@ function renderHistory(): void {
for (const msg of messages) {
if (msg.role === 'user') {
renderUserBubble(messagesContainer, msg.content, msg.timestamp);
renderUserBubble(messagesContainer, msg.content, msg.timestamp, msg.images);
} else {
const wrapper = renderAIBubble(messagesContainer, msg.content, msg.timestamp, msg.id, msg.feedback);
if (msg.sources && msg.sources.length > 0) renderSources(wrapper, msg.sources);
@ -731,6 +810,7 @@ function handleClear(): void {
if (clearBtn) clearBtn.style.display = 'none';
updateEmptyState();
clearMessages(config.integrateId);
clearPendingImages();
// 生成新的 chatId,开始新会话
const newId = generateNewChatId();
@ -789,6 +869,7 @@ export async function switchRole(newRoleId: string): Promise<void> {
messages = [];
const msgNodes = messagesContainer.querySelectorAll('.csk-msg, .csk-loading');
msgNodes.forEach(el => el.remove());
clearPendingImages();
// 5. 清空新旧角色的 localStorage 缓存(消息 + chatId),必须在 setActiveRoleId 之前
clearMessages(oldRoleId);
@ -928,6 +1009,7 @@ async function switchToConversation(conversationId: string): Promise<void> {
messages = [];
const msgs = messagesContainer.querySelectorAll('.csk-msg, .csk-loading');
msgs.forEach(el => el.remove());
clearPendingImages();
// 4. 从后端加载该会话的消息
try {

1
client/src/config.ts

@ -87,6 +87,7 @@ export function parseConfig(raw: SDKConfig): ResolvedConfig | null {
rewriteStrategy: raw.rewriteStrategy || 'REWRITE',
locale: raw.locale || 'zh-CN',
debug: raw.debug ?? true,
allowImageUpload: raw.allowImageUpload ?? true,
sound: raw.sound ?? false,
notification: raw.notification ?? false,
onError: typeof raw.onError === 'function' ? raw.onError : undefined,

28
client/src/dom.ts

@ -7,7 +7,7 @@
* -
* -
*/
import { ResolvedConfig, RagSource } from './types';
import { ResolvedConfig, RagSource, ImageAttachment } from './types';
import { debounce, formatTime, formatHistoryTime } from './utils';
import { t } from './i18n';
@ -352,6 +352,8 @@ export function createChatWindow(config: ResolvedConfig): {
inputEl.setAttribute('placeholder', t('placeholder'));
// 压缩输入框:autosize 为 Object 复杂类型,走 property 赋值(同 dom.ts createChatItemMsg 的 content 处理)
(inputEl as unknown as { autosize: { minRows: number } }).autosize = { minRows: 1 };
// 启用附件按钮 + 发送按钮(actions 为 Array 复杂类型,走 property 赋值)
(inputEl as unknown as { actions: string[] }).actions = config.allowImageUpload ? ['attachment', 'send'] : ['send'];
inputArea.appendChild(inputEl);
// === 保密声明脚注(默认折叠为单行,点击展开完整条文) ===
@ -823,11 +825,31 @@ function createChatItemMsg(role: 'user' | 'assistant', content: unknown[], times
return msg;
}
export function renderUserBubble(container: HTMLElement, text: string, timestamp: number): HTMLElement {
export function renderUserBubble(
container: HTMLElement,
text: string,
timestamp: number,
images?: ImageAttachment[]
): HTMLElement {
const wrapper = document.createElement('div');
wrapper.className = 'csk-msg csk-msg--user';
wrapper.appendChild(createChatItemMsg('user', [{ type: 'text', data: text }], timestamp));
// 组装 content:文本 + 附件(TDesign Chat 的 attachment content 类型,复用组件原生渲染)
const content: Array<{ type: string; data: unknown }> = [];
if (text) content.push({ type: 'text', data: text });
if (images && images.length > 0) {
content.push({
type: 'attachment',
data: images.map(img => ({
name: img.name,
url: img.url,
fileType: 'image',
size: img.size,
})),
});
}
wrapper.appendChild(createChatItemMsg('user', content, timestamp));
container.appendChild(wrapper);

2
client/src/i18n.ts

@ -74,6 +74,7 @@ const dictionaries: Record<string, Record<string, string>> = {
error_unavailable: '服务暂不可用,请稍后重试',
error_unknown: '请求发生未知错误',
error_send: '发送失败,请稍后重试',
error_upload_failed: '图片上传失败,请稍后重试',
error_stream_unsupported: '浏览器不支持流式读取',
},
@ -147,6 +148,7 @@ const dictionaries: Record<string, Record<string, string>> = {
error_unavailable: 'Service temporarily unavailable',
error_unknown: 'Unknown request error',
error_send: 'Failed to send, please try again',
error_upload_failed: 'Image upload failed, please try again',
error_stream_unsupported: 'Browser does not support streaming',
},
};

1
client/src/index.ts

@ -28,6 +28,7 @@ export type {
SDKConfig,
ResolvedConfig,
ChatMessage,
ImageAttachment,
RagSource,
CategoryNode,
ChatbotSDKInstance,

20
client/src/types.ts

@ -83,6 +83,10 @@ export interface SDKConfig {
/** 是否输出调试日志,默认 true */
debug?: boolean;
// === 图片上传配置 ===
/** 是否允许上传图片参与对话,默认 true */
allowImageUpload?: boolean;
// === 通知配置 ===
/** 弹窗关闭时收到新消息是否播放提示音,默认 false */
sound?: boolean;
@ -168,6 +172,20 @@ export interface ResolvedConfig {
onMessage?: (msg: ChatMessage) => void;
/** 当前对话 ID(自动管理,从 /conversation/list 获取或自动生成) */
chatId: string;
/** 是否允许上传图片参与对话 */
allowImageUpload: boolean;
}
/** 图片附件 */
export interface ImageAttachment {
/** 文件名 */
name: string;
/** 可访问的 URL */
url: string;
/** MIME 类型 */
mimeType?: string;
/** 文件大小(字节) */
size?: number;
}
/** 聊天消息 */
@ -180,6 +198,8 @@ export interface ChatMessage {
content: string;
/** 时间戳(毫秒) */
timestamp: number;
/** 可选:用户消息携带的图片 */
images?: ImageAttachment[];
/** 可选:RAG 引用来源 */
sources?: RagSource[];
/** 可选:用户反馈 'up' | 'down',预留后端对接位 */

6
src/main/java/com/wok/supportbot/controller/AttachmentController.java

@ -3,7 +3,6 @@ package com.wok.supportbot.controller;
import com.wok.supportbot.config.StorageProperties;
import com.wok.supportbot.service.SftpStorageService;
import org.springframework.http.ResponseEntity;
import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
@ -35,11 +34,14 @@ public class AttachmentController {
/**
* 上传图片/附件
*
* <p>鉴权由 {@code SdkAuthFilter} 承担拦截 /attachment/upload验证 SDK JWT 或管理后台 JWT
* /ai/** 接口一致方法层不再叠加 @PreAuthorize否则 SDK Token 会被 JwtAuthFilter 用管理后台
* 密钥重复验证失败导致 isAuthenticated() 判定为未认证
*
* @param file 上传的文件
* @return 上传结果data 内含 url / name / type / mimeType / size
*/
@PostMapping("/upload")
@PreAuthorize("isAuthenticated()")
public ResponseEntity<Map<String, Object>> upload(@RequestParam("file") MultipartFile file) {
try {
validateFile(file);

4
src/main/java/com/wok/supportbot/security/JwtAuthFilter.java

@ -90,7 +90,9 @@ public class JwtAuthFilter extends OncePerRequestFilter {
|| path.startsWith("/assets/")
|| path.startsWith("/sdk/")
|| path.equals("/favicon.ico")
|| path.equals("/favicon.svg");
|| path.equals("/favicon.svg")
|| path.equals("/feedback")
|| path.equals("/attachment/upload");
}
/**

4
src/main/java/com/wok/supportbot/security/SdkAuthFilter.java

@ -71,8 +71,8 @@ public class SdkAuthFilter extends OncePerRequestFilter {
if (path.startsWith("/ai/system-config/")) {
return true;
}
// 拦截 /ai/ 路径 /feedback 端点排除静态资源和 swagger
return !path.startsWith("/ai/") && !path.equals("/feedback");
// 拦截 /ai/ 路径/feedback 端点以及 SDK 上传接口 /attachment/upload排除静态资源和 swagger
return !path.startsWith("/ai/") && !path.equals("/feedback") && !path.equals("/attachment/upload");
}
@Override

2
src/main/java/com/wok/supportbot/security/SecurityConfig.java

@ -67,6 +67,8 @@ public class SecurityConfig {
.requestMatchers("/auth/login", "/auth/refresh", "/auth/logout").permitAll()
// SDK 需要的接口feedback分类会话查询等
.requestMatchers("/feedback").permitAll()
// 图片/附件上传实际鉴权由 SdkAuthFilter 承担 /ai/** 一致
.requestMatchers("/attachment/upload").permitAll()
.requestMatchers("/category/tree", "/category/list").permitAll()
// 系统配置SDK 拉取声明内容
.requestMatchers("/ai/system-config/**").permitAll()

2
src/main/resources/static/sdk/test.html

@ -11,8 +11,8 @@
<link rel="modulepreload" crossorigin href="/assets/markdown-pxFwF6hE.js">
<link rel="modulepreload" crossorigin href="/assets/chatAdapter-DJkCuBX0.js">
<link rel="stylesheet" crossorigin href="/assets/tdesign-CY0HVqZ3.css">
<link rel="stylesheet" crossorigin href="/assets/tdesign-web-components-B-ycfzW_.css">
<link rel="stylesheet" crossorigin href="/assets/tdesign-chat-Dj1Q23QO.css">
<link rel="stylesheet" crossorigin href="/assets/tdesign-web-components-B-ycfzW_.css">
<link rel="stylesheet" crossorigin href="/assets/sdk-test-wRid8JYa.css">
</head>
<body>

Loading…
Cancel
Save