diff --git a/client/src/api.ts b/client/src/api.ts index 6971906..118db1d 100644 --- a/client/src/api.ts +++ b/client/src/api.ts @@ -649,7 +649,7 @@ export async function fetchConversationList( * 使用 /ai/sdk/conversation/{id}/messages,含账户归属校验 */ export async function fetchConversationMessages(conversationId: string): Promise<{ - messages: Array<{ messageType: string; content: string; createTime: string }>; + messages: Array<{ messageType: string; content: string; createTime: string; images?: ImageAttachment[] }>; total: number; }> { const params = new URLSearchParams(); diff --git a/client/src/chat.ts b/client/src/chat.ts index 1e39fc7..eb08893 100644 --- a/client/src/chat.ts +++ b/client/src/chat.ts @@ -181,6 +181,12 @@ async function loadHistoryFromBackend(): Promise { id: uuid(), role: msg.messageType === 'USER' ? 'user' : 'ai' as const, content: msg.content, + images: (msg.images || []).map((i) => ({ + name: i.name || '', + url: i.url, + mimeType: i.mimeType, + size: i.size, + })), timestamp: new Date(msg.createTime).getTime(), })); @@ -1019,6 +1025,12 @@ async function switchToConversation(conversationId: string): Promise { id: uuid(), role: msg.messageType === 'USER' ? 'user' : 'ai' as const, content: msg.content, + images: (msg.images || []).map((i) => ({ + name: i.name || '', + url: i.url, + mimeType: i.mimeType, + size: i.size, + })), timestamp: new Date(msg.createTime).getTime(), })); diff --git a/src/main/java/com/wok/supportbot/converter/MessageConverter.java b/src/main/java/com/wok/supportbot/converter/MessageConverter.java index d4b93b8..45b6c73 100644 --- a/src/main/java/com/wok/supportbot/converter/MessageConverter.java +++ b/src/main/java/com/wok/supportbot/converter/MessageConverter.java @@ -2,6 +2,9 @@ package com.wok.supportbot.converter; import com.wok.supportbot.entity.ChatMessage; import org.springframework.ai.chat.messages.*; +import org.springframework.ai.content.Media; +import org.springframework.util.MimeType; +import org.springframework.util.MimeTypeUtils; import java.util.*; @@ -16,6 +19,8 @@ public class MessageConverter { private static final String META_KEY_TOOL_CALLS = "_toolCalls"; /** metadata 中存储 toolResponses 的内部键名 */ private static final String META_KEY_TOOL_RESPONSES = "_toolResponses"; + /** metadata 中存储多模态图片 Media 的内部键名 */ + private static final String META_KEY_MEDIA = "_media"; /** * 将 Spring AI Message 转换为数据库实体 ChatMessage @@ -54,6 +59,26 @@ public class MessageConverter { metadata.put(META_KEY_TOOL_RESPONSES, responsesJson); } + // 处理多模态图片 Media(仅用户消息携带),序列化到 metadata 供历史会话还原图片附件 + if (message instanceof UserMessage userMessage) { + List mediaList = userMessage.getMedia(); + if (mediaList != null && !mediaList.isEmpty()) { + List> mediaJson = new ArrayList<>(); + for (Media media : mediaList) { + Map m = new LinkedHashMap<>(); + Object data = media.getData(); + if (data != null) { + m.put("url", String.valueOf(data)); + } + if (media.getMimeType() != null) { + m.put("mimeType", media.getMimeType().toString()); + } + mediaJson.add(m); + } + metadata.put(META_KEY_MEDIA, mediaJson); + } + } + return ChatMessage.builder() .conversationId(conversationId) .messageType(message.getMessageType()) @@ -78,7 +103,11 @@ public class MessageConverter { : new HashMap<>(); return switch (messageType) { - case USER -> new UserMessage(text); + case USER -> { + List media = extractMedia(cleanMetadata); + yield media.isEmpty() ? new UserMessage(text) + : UserMessage.builder().text(text).media(media).build(); + } case ASSISTANT -> { List toolCalls = extractToolCalls(cleanMetadata); yield new AssistantMessage(text, cleanMetadata, toolCalls); @@ -135,4 +164,43 @@ public class MessageConverter { } return result; } + + /** + * 从 metadata 中提取并移除 _media 键,反序列化为 Media 列表。 + * 无媒体时返回空列表,保证多轮记忆不丢图片。 + */ + @SuppressWarnings("unchecked") + private static List extractMedia(Map metadata) { + Object raw = metadata.remove(META_KEY_MEDIA); + if (!(raw instanceof List list) || list.isEmpty()) { + return List.of(); + } + List result = new ArrayList<>(); + for (Object item : list) { + if (item instanceof Map) { + Map map = (Map) item; + Object url = map.get("url"); + if (url == null) { + continue; + } + result.add(Media.builder() + .data(url.toString()) + .mimeType(parseMimeTypeSafely(map.get("mimeType"))) + .build()); + } + } + return result; + } + + /** 安全解析 MIME 类型,缺失或非法时兜底为 image/png(与图片扩展名推断逻辑一致) */ + private static MimeType parseMimeTypeSafely(Object mimeType) { + if (mimeType == null) { + return MimeTypeUtils.IMAGE_PNG; + } + try { + return MimeTypeUtils.parseMimeType(mimeType.toString()); + } catch (Exception e) { + return MimeTypeUtils.IMAGE_PNG; + } + } } diff --git a/src/main/java/com/wok/supportbot/service/ConversationService.java b/src/main/java/com/wok/supportbot/service/ConversationService.java index f51a164..0b37a7c 100644 --- a/src/main/java/com/wok/supportbot/service/ConversationService.java +++ b/src/main/java/com/wok/supportbot/service/ConversationService.java @@ -283,7 +283,14 @@ public class ConversationService { item.put("conversationId", msg.getConversationId()); item.put("messageType", msg.getMessageType()); item.put("content", msg.getContent()); - item.put("metadata", msg.getMetadata()); + // 复制 metadata,提取内部键 _media 为顶层 images 字段(供历史会话还原图片附件),避免内部键泄漏 + Map metadata = msg.getMetadata() != null + ? new LinkedHashMap<>(msg.getMetadata()) : new LinkedHashMap<>(); + Object media = metadata.remove("_media"); + item.put("metadata", metadata); + if (media instanceof List mediaList && !mediaList.isEmpty()) { + item.put("images", mediaList); + } item.put("createTime", msg.getCreateTime()); result.add(item); }