4 changed files with 235 additions and 26 deletions
-
28CLAUDE.md
-
38src/main/java/com/wok/supportbot/config/EmbeddingModelFactory.java
-
189src/main/java/com/wok/supportbot/config/VolcengineMultimodalEmbeddingModel.java
-
6src/main/resources/static/components/ModelConfigManager.js
@ -0,0 +1,189 @@ |
|||||
|
package com.wok.supportbot.config; |
||||
|
|
||||
|
import com.fasterxml.jackson.core.type.TypeReference; |
||||
|
import com.fasterxml.jackson.databind.ObjectMapper; |
||||
|
import lombok.extern.slf4j.Slf4j; |
||||
|
import org.springframework.ai.document.Document; |
||||
|
import org.springframework.ai.embedding.Embedding; |
||||
|
import org.springframework.ai.embedding.EmbeddingModel; |
||||
|
import org.springframework.ai.embedding.EmbeddingRequest; |
||||
|
import org.springframework.ai.embedding.EmbeddingResponse; |
||||
|
import org.springframework.ai.embedding.EmbeddingResponseMetadata; |
||||
|
import org.springframework.ai.chat.metadata.DefaultUsage; |
||||
|
import org.springframework.http.HttpHeaders; |
||||
|
import org.springframework.http.MediaType; |
||||
|
import org.springframework.retry.support.RetryTemplate; |
||||
|
import org.springframework.web.client.RestClient; |
||||
|
|
||||
|
import java.util.ArrayList; |
||||
|
import java.util.HashMap; |
||||
|
import java.util.List; |
||||
|
import java.util.Map; |
||||
|
|
||||
|
/** |
||||
|
* 豆包多模态向量化模型（Volcengine Multimodal Embedding Model） |
||||
|
* <p> |
||||
|
* 直接调用豆包 /embeddings/multimodal 端点，适配其特有的请求/响应格式。 |
||||
|
* <p> |
||||
|
* 与 OpenAI Embedding API 的核心差异： |
||||
|
* - 端点: /embeddings/multimodal（非 /embeddings） |
||||
|
* - 请求 input: [{type, text}] 对象数组（非纯字符串数组） |
||||
|
* - 响应 data: 单个对象 {"embedding": [...]}（非数组 [{...}]） |
||||
|
* - 语义: 整个 input 数组被视为一个文档，只返回一个 embedding 向量 |
||||
|
* <p> |
||||
|
* 因此必须逐条调用：每条文本单独发一次请求，不能批量。 |
||||
|
* 当前阶段仅处理纯文本（type=text），未来可扩展图片/视频。 |
||||
|
*/ |
||||
|
@Slf4j |
||||
|
public class VolcengineMultimodalEmbeddingModel implements EmbeddingModel { |
||||
|
|
||||
|
private static final String EMBEDDINGS_PATH = "/embeddings/multimodal"; |
||||
|
|
||||
|
private final String apiKey; |
||||
|
private final String baseUrl; |
||||
|
private final String modelName; |
||||
|
private final int dimensions; |
||||
|
private final RetryTemplate retryTemplate; |
||||
|
private final RestClient restClient; |
||||
|
private final ObjectMapper objectMapper = new ObjectMapper(); |
||||
|
|
||||
|
public VolcengineMultimodalEmbeddingModel(String apiKey, String baseUrl, String modelName, |
||||
|
int dimensions, RetryTemplate retryTemplate) { |
||||
|
this.apiKey = apiKey; |
||||
|
this.baseUrl = baseUrl; |
||||
|
this.modelName = modelName; |
||||
|
this.dimensions = dimensions; |
||||
|
this.retryTemplate = retryTemplate; |
||||
|
this.restClient = RestClient.builder() |
||||
|
.baseUrl(baseUrl) |
||||
|
.defaultHeader(HttpHeaders.AUTHORIZATION, "Bearer " + apiKey) |
||||
|
.defaultHeader(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_JSON_VALUE) |
||||
|
.build(); |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public float[] embed(Document document) { |
||||
|
if (document.isText()) { |
||||
|
return embedSingleText(document.getText()); |
||||
|
} |
||||
|
// 未来扩展：检查 document.getMedia() 处理图片/视频 |
||||
|
throw new IllegalArgumentException( |
||||
|
"豆包多模态 Embedding 暂不支持非文本内容，Document 需为纯文本"); |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public EmbeddingResponse call(EmbeddingRequest request) { |
||||
|
List<String> texts = request.getInstructions(); |
||||
|
|
||||
|
// 豆包 /embeddings/multimodal 把整个 input 数组视为一个文档，只返回一个 embedding。 |
||||
|
// 因此必须逐条调用，每条文本单独发一次请求。 |
||||
|
List<Embedding> embeddings = new ArrayList<>(); |
||||
|
int totalPromptTokens = 0; |
||||
|
int totalTotalTokens = 0; |
||||
|
|
||||
|
for (int i = 0; i < texts.size(); i++) { |
||||
|
final int index = i; |
||||
|
final String text = texts.get(i); |
||||
|
|
||||
|
// 构造单条文本的多模态 input |
||||
|
Map<String, Object> inputItem = new HashMap<>(); |
||||
|
inputItem.put("type", "text"); |
||||
|
inputItem.put("text", text); |
||||
|
|
||||
|
Map<String, Object> requestBody = new HashMap<>(); |
||||
|
requestBody.put("model", modelName); |
||||
|
requestBody.put("input", List.of(inputItem)); |
||||
|
// 豆包 vision 模型默认输出 2048 维，超过 PGVector HNSW 索引上限 2000。 |
||||
|
// 传入 dimensions 参数让 API 端降维（支持 Matryoshka 降维到 1024 等） |
||||
|
requestBody.put("dimensions", dimensions); |
||||
|
|
||||
|
log.debug("豆包多模态 Embedding 请求: model={}, index={}/{}", modelName, index + 1, texts.size()); |
||||
|
|
||||
|
EmbeddingResult result = retryTemplate.execute(ctx -> { |
||||
|
String responseBody = restClient.post() |
||||
|
.uri(EMBEDDINGS_PATH) |
||||
|
.body(requestBody) |
||||
|
.retrieve() |
||||
|
.body(String.class); |
||||
|
|
||||
|
return parseResponse(responseBody); |
||||
|
}); |
||||
|
|
||||
|
embeddings.add(new Embedding(result.embedding, index)); |
||||
|
|
||||
|
// 累计 usage |
||||
|
if (result.promptTokens > 0) { |
||||
|
totalPromptTokens += result.promptTokens; |
||||
|
totalTotalTokens += result.totalTokens; |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
EmbeddingResponseMetadata metadata = new EmbeddingResponseMetadata(modelName, |
||||
|
new DefaultUsage(totalPromptTokens, 0, totalTotalTokens, null)); |
||||
|
|
||||
|
return new EmbeddingResponse(embeddings, metadata); |
||||
|
} |
||||
|
|
||||
|
/** |
||||
|
* 单条文本向量化（供 embed(Document) 使用） |
||||
|
*/ |
||||
|
private float[] embedSingleText(String text) { |
||||
|
EmbeddingResponse response = call(new EmbeddingRequest(List.of(text), null)); |
||||
|
List<Embedding> results = response.getResults(); |
||||
|
return results.isEmpty() ? new float[0] : results.get(0).getOutput(); |
||||
|
} |
||||
|
|
||||
|
/** |
||||
|
* 解析豆包 /embeddings/multimodal 响应。 |
||||
|
* 响应格式：{"created":..., "data":{"embedding":[0.1,0.2,...]}, "id":"...", "model":"...", "object":"...", "usage":{...}} |
||||
|
* 注意：data 是单个对象（非数组），embedding 值可能是 Integer 或 Double。 |
||||
|
*/ |
||||
|
private EmbeddingResult parseResponse(String responseBody) { |
||||
|
try { |
||||
|
Map<String, Object> body = objectMapper.readValue(responseBody, |
||||
|
new TypeReference<Map<String, Object>>() {}); |
||||
|
|
||||
|
// 解析 data.embedding — data 是单个对象 {"embedding": [...]} |
||||
|
@SuppressWarnings("unchecked") |
||||
|
Map<String, Object> data = (Map<String, Object>) body.get("data"); |
||||
|
@SuppressWarnings("unchecked") |
||||
|
List<Number> rawEmbedding = (List<Number>) data.get("embedding"); |
||||
|
|
||||
|
float[] vec = new float[rawEmbedding.size()]; |
||||
|
for (int i = 0; i < rawEmbedding.size(); i++) { |
||||
|
vec[i] = rawEmbedding.get(i).floatValue(); |
||||
|
} |
||||
|
|
||||
|
// 解析 usage |
||||
|
int promptTokens = 0; |
||||
|
int totalTokens = 0; |
||||
|
@SuppressWarnings("unchecked") |
||||
|
Map<String, Object> usage = (Map<String, Object>) body.get("usage"); |
||||
|
if (usage != null) { |
||||
|
promptTokens = ((Number) usage.getOrDefault("prompt_tokens", 0)).intValue(); |
||||
|
totalTokens = ((Number) usage.getOrDefault("total_tokens", 0)).intValue(); |
||||
|
} |
||||
|
|
||||
|
return new EmbeddingResult(vec, promptTokens, totalTokens); |
||||
|
} catch (com.fasterxml.jackson.core.JsonProcessingException e) { |
||||
|
throw new RuntimeException("解析豆包多模态 Embedding 响应失败: " + e.getMessage() |
||||
|
+ ", 响应体前500字符: " + responseBody.substring(0, Math.min(500, responseBody.length())), e); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public EmbeddingResponse embedForResponse(List<String> texts) { |
||||
|
return call(new EmbeddingRequest(texts, null)); |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public int dimensions() { |
||||
|
return dimensions; |
||||
|
} |
||||
|
|
||||
|
/** |
||||
|
* 内部解析结果 |
||||
|
*/ |
||||
|
private record EmbeddingResult(float[] embedding, int promptTokens, int totalTokens) { |
||||
|
} |
||||
|
} |
||||
Write
Preview
Loading…
Cancel
Save
Reference in new issue