diff --git a/mateclaw-server/src/main/java/vip/mate/agent/AgentGraphBuilder.java b/mateclaw-server/src/main/java/vip/mate/agent/AgentGraphBuilder.java index eb252a7e..0519bf1f 100644 --- a/mateclaw-server/src/main/java/vip/mate/agent/AgentGraphBuilder.java +++ b/mateclaw-server/src/main/java/vip/mate/agent/AgentGraphBuilder.java @@ -675,7 +675,10 @@ public class AgentGraphBuilder { // RFC-062: Claude Code OAuth tunnels through the same Messages API // wrapped in AnthropicChatModel — same StateGraph capability surface. || protocol == ModelProtocol.ANTHROPIC_CLAUDE_CODE - || protocol == ModelProtocol.OPENAI_CHATGPT; + || protocol == ModelProtocol.OPENAI_CHATGPT + // Gemini native generateContent — GeminiChatModel exposes the same + // streaming + tool-calling surface the StateGraph nodes rely on. + || protocol == ModelProtocol.GEMINI_NATIVE; } // ==================== 模型构建 ==================== diff --git a/mateclaw-server/src/main/java/vip/mate/llm/chatmodel/GeminiChatModelBuilder.java b/mateclaw-server/src/main/java/vip/mate/llm/chatmodel/GeminiChatModelBuilder.java new file mode 100644 index 00000000..32fc52d9 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/llm/chatmodel/GeminiChatModelBuilder.java @@ -0,0 +1,51 @@ +package vip.mate.llm.chatmodel; + +import lombok.RequiredArgsConstructor; +import org.springframework.ai.chat.model.ChatModel; +import org.springframework.retry.support.RetryTemplate; +import org.springframework.stereotype.Component; +import vip.mate.exception.MateClawException; +import vip.mate.llm.gemini.GeminiChatModel; +import vip.mate.llm.gemini.GeminiNativeClient; +import vip.mate.llm.model.ModelConfigEntity; +import vip.mate.llm.model.ModelProtocol; +import vip.mate.llm.model.ModelProviderEntity; +import vip.mate.llm.service.ModelProviderService; + +/** + * Strategy implementation for {@link ModelProtocol#GEMINI_NATIVE}. + * + *

Builds a {@link GeminiChatModel} over the native Gemini + * {@code generateContent} API. The {@link RetryTemplate} is ignored — the + * native client owns its own HTTP transport, mirroring the ChatGPT Responses + * builder.

+ */ +@Component +@RequiredArgsConstructor +public class GeminiChatModelBuilder implements ChatModelBuilder { + + private final GeminiNativeClient geminiNativeClient; + private final ModelProviderService modelProviderService; + + @Override + public ModelProtocol supportedProtocol() { + return ModelProtocol.GEMINI_NATIVE; + } + + @Override + public ChatModel build(ModelConfigEntity model, ModelProviderEntity provider, RetryTemplate retry) { + if (provider == null || !modelProviderService.isProviderConfigured(provider.getProviderId())) { + throw new MateClawException("err.agent.gemini_not_configured", + "Gemini Provider 未完成配置,请在模型设置中填写有效的 API Key"); + } + String apiKey = provider.getApiKey(); + if (!modelProviderService.hasUsableApiKey(apiKey)) { + throw new MateClawException("err.agent.gemini_key_invalid", + "Gemini API Key 未配置或无效: " + provider.getProviderId()); + } + Double temperature = model.getTemperature(); + Integer maxTokens = model.getMaxTokens(); + return new GeminiChatModel(geminiNativeClient, provider.getBaseUrl(), apiKey.trim(), + model.getModelName(), temperature, maxTokens); + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/llm/failover/probe/GeminiListModelsProbe.java b/mateclaw-server/src/main/java/vip/mate/llm/failover/probe/GeminiListModelsProbe.java new file mode 100644 index 00000000..635847eb --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/llm/failover/probe/GeminiListModelsProbe.java @@ -0,0 +1,96 @@ +package vip.mate.llm.failover.probe; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.http.HttpHeaders; +import org.springframework.http.MediaType; +import org.springframework.http.client.JdkClientHttpRequestFactory; +import org.springframework.stereotype.Component; +import org.springframework.util.StringUtils; +import org.springframework.web.client.HttpClientErrorException; +import org.springframework.web.client.HttpServerErrorException; +import org.springframework.web.client.RestClient; +import vip.mate.llm.failover.ProbeResult; +import vip.mate.llm.failover.ProviderProbeStrategy; +import vip.mate.llm.model.ModelProtocol; +import vip.mate.llm.model.ModelProviderEntity; + +import java.net.http.HttpClient; +import java.time.Duration; + +/** + * Probes a native Gemini provider by listing its models via + * {@code GET /v1beta/models?key=...}. + * + *

Auth failures (400 with an API-key error, 401, 403) are definitive + * negatives → HARD remove. Other 4xx/5xx are treated fail-open, since the + * chat path is the authoritative check.

+ */ +@Slf4j +@Component +public class GeminiListModelsProbe implements ProviderProbeStrategy { + + private static final Duration TIMEOUT = Duration.ofSeconds(5); + + @Override + public ModelProtocol supportedProtocol() { + return ModelProtocol.GEMINI_NATIVE; + } + + @Override + public ProbeResult probe(ModelProviderEntity provider) { + if (provider == null || !StringUtils.hasText(provider.getApiKey())) { + return ProbeResult.fail(0, "API key not configured"); + } + String baseUrl = provider.getBaseUrl(); + if (!StringUtils.hasText(baseUrl)) { + baseUrl = "https://generativelanguage.googleapis.com"; + } + baseUrl = stripTrailingSlash(baseUrl.trim()); + long start = System.currentTimeMillis(); + try { + HttpClient httpClient = HttpClient.newBuilder().connectTimeout(TIMEOUT).build(); + RestClient client = RestClient.builder() + .baseUrl(baseUrl) + .requestFactory(new JdkClientHttpRequestFactory(httpClient)) + .defaultHeader(HttpHeaders.ACCEPT, MediaType.APPLICATION_JSON_VALUE) + .build(); + client.get() + .uri("/v1beta/models?key={key}", provider.getApiKey().trim()) + .retrieve() + .body(String.class); + return ProbeResult.ok(System.currentTimeMillis() - start); + } catch (HttpClientErrorException e) { + long latency = System.currentTimeMillis() - start; + int status = e.getStatusCode().value(); + if (status == 401 || status == 403 + || (status == 400 && e.getResponseBodyAsString().contains("API key"))) { + log.debug("[Probe] {} gemini models auth failed: {}", provider.getProviderId(), status); + return ProbeResult.fail(latency, "auth failed (" + status + ")"); + } + log.info("[Probe] {} gemini models returned {} — fail-open (inconclusive)", + provider.getProviderId(), status); + return ProbeResult.ok(latency); + } catch (HttpServerErrorException e) { + long latency = System.currentTimeMillis() - start; + log.info("[Probe] {} gemini models returned {} — fail-open (5xx may be transient)", + provider.getProviderId(), e.getStatusCode().value()); + return ProbeResult.ok(latency); + } catch (Exception e) { + long latency = System.currentTimeMillis() - start; + log.debug("[Probe] {} gemini models failed: {}", provider.getProviderId(), e.getMessage()); + return ProbeResult.fail(latency, shortMessage(e)); + } + } + + private static String stripTrailingSlash(String url) { + return url.endsWith("/") ? url.substring(0, url.length() - 1) : url; + } + + private static String shortMessage(Throwable t) { + String m = t.getMessage(); + if (m == null) { + m = t.getClass().getSimpleName(); + } + return m.length() > 200 ? m.substring(0, 200) + "..." : m; + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/llm/gemini/GeminiChatModel.java b/mateclaw-server/src/main/java/vip/mate/llm/gemini/GeminiChatModel.java new file mode 100644 index 00000000..12a8a191 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/llm/gemini/GeminiChatModel.java @@ -0,0 +1,223 @@ +package vip.mate.llm.gemini; + +import com.fasterxml.jackson.databind.JsonNode; +import lombok.extern.slf4j.Slf4j; +import org.springframework.ai.chat.messages.AssistantMessage; +import org.springframework.ai.chat.messages.Message; +import org.springframework.ai.chat.metadata.ChatGenerationMetadata; +import org.springframework.ai.chat.metadata.ChatResponseMetadata; +import org.springframework.ai.chat.metadata.DefaultUsage; +import org.springframework.ai.chat.metadata.Usage; +import org.springframework.ai.chat.model.ChatModel; +import org.springframework.ai.chat.model.ChatResponse; +import org.springframework.ai.chat.model.Generation; +import org.springframework.ai.chat.prompt.ChatOptions; +import org.springframework.ai.chat.prompt.Prompt; +import org.springframework.ai.model.tool.ToolCallingChatOptions; +import org.springframework.ai.tool.ToolCallback; +import org.springframework.ai.tool.definition.ToolDefinition; +import reactor.core.publisher.Flux; +import vip.mate.llm.gemini.GeminiNativeClient.GeminiCall; +import vip.mate.llm.gemini.GeminiNativeClient.StreamEvent; + +import java.util.ArrayList; +import java.util.List; + +/** + * Spring AI {@link ChatModel} backed by the native Gemini + * {@code generateContent} API. Extracts tool callbacks from the prompt + * options, delegates message translation + transport to + * {@link GeminiNativeClient}, and adapts Gemini's whole-at-once function calls + * into the start/args-delta {@link ChatResponse} shape the streaming agent + * pipeline expects. + */ +@Slf4j +public class GeminiChatModel implements ChatModel { + + private final GeminiNativeClient client; + private final String baseUrl; + private final String apiKey; + private final String modelName; + private final Double temperature; + private final Integer maxTokens; + + public GeminiChatModel(GeminiNativeClient client, String baseUrl, String apiKey, + String modelName, Double temperature, Integer maxTokens) { + this.client = client; + this.baseUrl = baseUrl; + this.apiKey = apiKey; + this.modelName = modelName; + this.temperature = temperature; + this.maxTokens = maxTokens; + } + + @Override + public ChatResponse call(Prompt prompt) { + GeminiCall call = buildCall(prompt); + JsonNode response = client.generate(call); + + StringBuilder text = new StringBuilder(); + List toolCalls = new ArrayList<>(); + String finishReason = "stop"; + + for (JsonNode candidate : response.path("candidates")) { + String reason = candidate.path("finishReason").asText(""); + if (!reason.isBlank()) { + finishReason = reason; + } + for (JsonNode part : candidate.path("content").path("parts")) { + if (part.path("thought").asBoolean(false)) { + continue; + } + JsonNode functionCall = part.get("functionCall"); + if (functionCall != null && !functionCall.isNull()) { + String name = functionCall.path("name").asText(""); + String id = functionCall.has("id") + ? functionCall.get("id").asText() + : "call_" + name + "_" + System.nanoTime(); + String args = functionCall.has("args") + ? functionCall.get("args").toString() : "{}"; + toolCalls.add(new AssistantMessage.ToolCall(id, "function", name, args)); + continue; + } + JsonNode partText = part.get("text"); + if (partText != null && partText.isTextual()) { + text.append(partText.asText()); + } + } + } + + AssistantMessage assistantMessage = toolCalls.isEmpty() + ? new AssistantMessage(text.toString()) + : AssistantMessage.builder().content(text.toString()).toolCalls(toolCalls).build(); + Generation generation = new Generation(assistantMessage, + ChatGenerationMetadata.builder().finishReason(finishReason).build()); + + ChatResponseMetadata.Builder metadata = ChatResponseMetadata.builder().model(call.model()); + Usage usage = extractUsage(response.get("usageMetadata")); + if (usage != null) { + metadata.usage(usage); + } + return new ChatResponse(List.of(generation), metadata.build()); + } + + @Override + public Flux stream(Prompt prompt) { + GeminiCall call = buildCall(prompt); + return client.streamEvents(call) + .concatMapIterable(event -> toChatResponses(event, call.model())); + } + + @Override + public ChatOptions getDefaultOptions() { + return ChatOptions.builder() + .model(modelName) + .temperature(temperature) + .build(); + } + + /** + * Adapt one {@link StreamEvent} into 0-2 {@link ChatResponse}s. A tool call + * becomes a start frame (id + name, empty args) followed by an args-delta + * frame (empty id, full args) so the streaming accumulator builds it the + * same way it does for OpenAI's incremental tool calls. + */ + private List toChatResponses(StreamEvent event, String model) { + switch (event.type()) { + case "text" -> { + Generation gen = new Generation(new AssistantMessage(event.text()), + ChatGenerationMetadata.builder().finishReason(null).build()); + return List.of(new ChatResponse(List.of(gen), + ChatResponseMetadata.builder().model(model).build())); + } + case "tool_call" -> { + AssistantMessage startMsg = AssistantMessage.builder() + .content("") + .toolCalls(List.of(new AssistantMessage.ToolCall( + event.toolCallId(), "function", event.toolName(), ""))) + .build(); + AssistantMessage deltaMsg = AssistantMessage.builder() + .content("") + .toolCalls(List.of(new AssistantMessage.ToolCall( + "", "function", "", event.toolArgs()))) + .build(); + ChatResponseMetadata md = ChatResponseMetadata.builder().model(model).build(); + return List.of( + new ChatResponse(List.of(new Generation(startMsg, + ChatGenerationMetadata.builder().finishReason(null).build())), md), + new ChatResponse(List.of(new Generation(deltaMsg, + ChatGenerationMetadata.builder().finishReason(null).build())), md)); + } + case "done" -> { + int in = event.inputTokens() != null ? event.inputTokens() : 0; + int out = event.outputTokens() != null ? event.outputTokens() : 0; + int total = event.totalTokens() != null ? event.totalTokens() : (in + out); + Usage usage = new DefaultUsage(in, out, total); + Generation gen = new Generation(new AssistantMessage(""), + ChatGenerationMetadata.builder().finishReason("stop").build()); + return List.of(new ChatResponse(List.of(gen), + ChatResponseMetadata.builder().model(model).usage(usage).build())); + } + default -> { + return List.of(); + } + } + } + + private GeminiCall buildCall(Prompt prompt) { + List messages = prompt.getInstructions(); + String model = resolveModel(prompt); + Double temp = resolveTemperature(prompt); + List tools = extractToolDefinitions(prompt); + return new GeminiCall(baseUrl, apiKey, model, messages, temp, maxTokens, tools); + } + + private Usage extractUsage(JsonNode usageMetadata) { + if (usageMetadata == null || usageMetadata.isNull()) { + return null; + } + int in = usageMetadata.path("promptTokenCount").asInt(0); + int out = usageMetadata.path("candidatesTokenCount").asInt(0); + int total = usageMetadata.has("totalTokenCount") + ? usageMetadata.get("totalTokenCount").asInt() : in + out; + return new DefaultUsage(in, out, total); + } + + private List extractToolDefinitions(Prompt prompt) { + ChatOptions options = prompt.getOptions(); + if (options == null) { + return List.of(); + } + List callbacks = null; + if (options instanceof ToolCallingChatOptions tcOpts) { + callbacks = tcOpts.getToolCallbacks(); + } else { + try { + var method = options.getClass().getMethod("getToolCallbacks"); + @SuppressWarnings("unchecked") + var result = (List) method.invoke(options); + callbacks = result; + } catch (Exception ignored) { + // options type carries no tool callbacks + } + } + if (callbacks == null || callbacks.isEmpty()) { + return List.of(); + } + return callbacks.stream().map(ToolCallback::getToolDefinition).toList(); + } + + private String resolveModel(Prompt prompt) { + if (prompt.getOptions() != null && prompt.getOptions().getModel() != null) { + return prompt.getOptions().getModel(); + } + return modelName; + } + + private Double resolveTemperature(Prompt prompt) { + if (prompt.getOptions() != null && prompt.getOptions().getTemperature() != null) { + return prompt.getOptions().getTemperature(); + } + return temperature; + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/llm/gemini/GeminiNativeClient.java b/mateclaw-server/src/main/java/vip/mate/llm/gemini/GeminiNativeClient.java new file mode 100644 index 00000000..6d05607b --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/llm/gemini/GeminiNativeClient.java @@ -0,0 +1,404 @@ +package vip.mate.llm.gemini; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.ai.chat.messages.AssistantMessage; +import org.springframework.ai.chat.messages.Message; +import org.springframework.ai.chat.messages.MessageType; +import org.springframework.ai.chat.messages.ToolResponseMessage; +import org.springframework.ai.chat.messages.UserMessage; +import org.springframework.ai.content.Media; +import org.springframework.ai.tool.definition.ToolDefinition; +import org.springframework.http.MediaType; +import org.springframework.stereotype.Component; +import org.springframework.web.reactive.function.client.WebClient; +import reactor.core.publisher.Flux; +import vip.mate.exception.MateClawException; + +import java.util.ArrayList; +import java.util.Base64; +import java.util.List; + +/** + * Native client for the Gemini {@code generateContent} API + * ({@code generativelanguage.googleapis.com}). + * + *

Translates Spring AI {@link Message} lists into Gemini {@code contents} + * (with {@code systemInstruction}, {@code functionCall} / {@code functionResponse} + * parts and inline image data), and parses the {@code streamGenerateContent} + * SSE stream back into {@link StreamEvent}s. Mirrors the role this project's + * {@code ChatGPTResponsesClient} plays for the OpenAI Responses API.

+ */ +@Slf4j +@Component +@RequiredArgsConstructor +public class GeminiNativeClient { + + private final ObjectMapper objectMapper; + private final WebClient webClient = WebClient.create(); + + /** + * One streamed delta from Gemini. {@code text} carries an output-text + * chunk; {@code tool_call} carries a complete function call (Gemini does + * not stream call arguments incrementally); {@code done} carries final + * token usage. + */ + public record StreamEvent(String type, String text, String toolCallId, String toolName, + String toolArgs, Integer inputTokens, Integer outputTokens, + Integer totalTokens) { + public static StreamEvent text(String delta) { + return new StreamEvent("text", delta, null, null, null, null, null, null); + } + public static StreamEvent toolCall(String id, String name, String args) { + return new StreamEvent("tool_call", null, id, name, args, null, null, null); + } + public static StreamEvent done(Integer in, Integer out, Integer total) { + return new StreamEvent("done", null, null, null, null, in, out, total); + } + } + + /** Connection parameters resolved from the provider + model rows. */ + public record GeminiCall(String baseUrl, String apiKey, String model, List messages, + Double temperature, Integer maxTokens, List tools) { + } + + /** + * Stream a {@code generateContent} call. Emits text deltas, complete tool + * calls, and a terminal {@code done} event with token usage. + */ + public Flux streamEvents(GeminiCall call) { + ObjectNode body = buildRequestBody(call); + String url = endpoint(call, "streamGenerateContent") + "&alt=sse"; + log.info("[Gemini] stream: model={}, messages={}, tools={}", call.model(), + call.messages().size(), call.tools() != null ? call.tools().size() : 0); + + return webClient.post() + .uri(url) + .contentType(MediaType.APPLICATION_JSON) + .accept(MediaType.TEXT_EVENT_STREAM) + .bodyValue(body.toString()) + .retrieve() + .onStatus(status -> status.is4xxClientError() || status.is5xxServerError(), + response -> response.bodyToMono(String.class) + .defaultIfEmpty("") + .map(errorBody -> { + log.error("[Gemini] API error {}: {}", response.statusCode(), errorBody); + return new MateClawException("err.llm.gemini_error", + "Gemini API " + response.statusCode() + ": " + + extractErrorMessage(errorBody)); + })) + .bodyToFlux(String.class) + .filter(line -> line.startsWith("data:")) + .map(line -> line.startsWith("data: ") ? line.substring(6) : line.substring(5)) + .filter(line -> !line.isBlank()) + .concatMapIterable(this::parseChunk) + .onErrorMap(e -> e instanceof MateClawException ? e + : new MateClawException("err.llm.gemini_stream_failed", + "Gemini 流式调用失败: " + e.getMessage())); + } + + /** Non-streaming {@code generateContent} call — returns the parsed response. */ + public JsonNode generate(GeminiCall call) { + ObjectNode body = buildRequestBody(call); + String url = endpoint(call, "generateContent"); + log.info("[Gemini] call: model={}, messages={}, tools={}", call.model(), + call.messages().size(), call.tools() != null ? call.tools().size() : 0); + + String response = webClient.post() + .uri(url) + .contentType(MediaType.APPLICATION_JSON) + .bodyValue(body.toString()) + .retrieve() + .onStatus(status -> status.is4xxClientError() || status.is5xxServerError(), + resp -> resp.bodyToMono(String.class) + .defaultIfEmpty("") + .map(errorBody -> { + log.error("[Gemini] API error {}: {}", resp.statusCode(), errorBody); + return new MateClawException("err.llm.gemini_error", + "Gemini API " + resp.statusCode() + ": " + + extractErrorMessage(errorBody)); + })) + .bodyToMono(String.class) + .block(); + try { + return objectMapper.readTree(response == null ? "{}" : response); + } catch (Exception e) { + throw new MateClawException("err.llm.gemini_error", "Gemini 响应解析失败: " + e.getMessage()); + } + } + + private String endpoint(GeminiCall call, String method) { + String base = call.baseUrl(); + if (base == null || base.isBlank()) { + base = "https://generativelanguage.googleapis.com"; + } + if (base.endsWith("/")) { + base = base.substring(0, base.length() - 1); + } + return base + "/v1beta/models/" + call.model() + ":" + method + "?key=" + call.apiKey(); + } + + // ==================== request building ==================== + + ObjectNode buildRequestBody(GeminiCall call) { + ObjectNode body = objectMapper.createObjectNode(); + + // System instruction — first SYSTEM message becomes systemInstruction. + for (Message msg : call.messages()) { + if (msg.getMessageType() == MessageType.SYSTEM) { + String sys = msg.getText(); + if (sys != null && !sys.isBlank()) { + ObjectNode systemInstruction = objectMapper.createObjectNode(); + ArrayNode sysParts = systemInstruction.putArray("parts"); + sysParts.addObject().put("text", sys); + body.set("systemInstruction", systemInstruction); + } + break; + } + } + + // contents — user / model / tool turns. + ArrayNode contents = body.putArray("contents"); + for (Message msg : call.messages()) { + switch (msg.getMessageType()) { + case SYSTEM -> { /* hoisted into systemInstruction */ } + case USER -> contents.add(buildUserContent(msg)); + case ASSISTANT -> { + ObjectNode modelContent = buildAssistantContent((AssistantMessage) msg); + if (modelContent != null) { + contents.add(modelContent); + } + } + case TOOL -> contents.add(buildToolContent((ToolResponseMessage) msg)); + } + } + + // tools — functionDeclarations. + if (call.tools() != null && !call.tools().isEmpty()) { + ArrayNode toolsArr = body.putArray("tools"); + ObjectNode toolEntry = toolsArr.addObject(); + ArrayNode declarations = toolEntry.putArray("functionDeclarations"); + for (ToolDefinition tool : call.tools()) { + ObjectNode decl = declarations.addObject(); + decl.put("name", tool.name()); + if (tool.description() != null) { + decl.put("description", tool.description()); + } + JsonNode params = parseSchema(tool.inputSchema()); + decl.set("parameters", GeminiSchemaSanitizer.sanitizeToolParameters(params, objectMapper)); + } + } + + // generationConfig. + ObjectNode genConfig = body.putObject("generationConfig"); + if (call.temperature() != null) { + genConfig.put("temperature", call.temperature()); + } + if (call.maxTokens() != null && call.maxTokens() > 0) { + genConfig.put("maxOutputTokens", call.maxTokens()); + } + return body; + } + + private ObjectNode buildUserContent(Message msg) { + ObjectNode content = objectMapper.createObjectNode(); + content.put("role", "user"); + ArrayNode parts = content.putArray("parts"); + String text = msg.getText(); + if (text != null && !text.isBlank()) { + parts.addObject().put("text", text); + } + if (msg instanceof UserMessage userMsg) { + for (Media media : userMsg.getMedia()) { + appendInlineMedia(parts, media); + } + } + if (parts.isEmpty()) { + parts.addObject().put("text", ""); + } + return content; + } + + private void appendInlineMedia(ArrayNode parts, Media media) { + try { + byte[] data = media.getDataAsByteArray(); + if (data == null || data.length == 0) { + return; + } + String mime = media.getMimeType() != null ? media.getMimeType().toString() : "image/png"; + ObjectNode inlineData = parts.addObject().putObject("inlineData"); + inlineData.put("mimeType", mime); + inlineData.put("data", Base64.getEncoder().encodeToString(data)); + } catch (Exception e) { + // URI-backed media (no inline bytes) — skip rather than fail the turn. + log.debug("[Gemini] skipping non-inline media: {}", e.getMessage()); + } + } + + private ObjectNode buildAssistantContent(AssistantMessage msg) { + ObjectNode content = objectMapper.createObjectNode(); + content.put("role", "model"); + ArrayNode parts = content.putArray("parts"); + String text = msg.getText(); + if (text != null && !text.isBlank()) { + parts.addObject().put("text", text); + } + if (msg.hasToolCalls()) { + for (AssistantMessage.ToolCall tc : msg.getToolCalls()) { + ObjectNode functionCall = parts.addObject().putObject("functionCall"); + functionCall.put("name", tc.name()); + functionCall.set("args", parseArgs(tc.arguments())); + } + } + if (parts.isEmpty()) { + return null; + } + return content; + } + + private ObjectNode buildToolContent(ToolResponseMessage msg) { + ObjectNode content = objectMapper.createObjectNode(); + content.put("role", "user"); + ArrayNode parts = content.putArray("parts"); + for (ToolResponseMessage.ToolResponse response : msg.getResponses()) { + ObjectNode functionResponse = parts.addObject().putObject("functionResponse"); + functionResponse.put("name", response.name()); + functionResponse.set("response", wrapToolResponse(response.responseData())); + } + if (parts.isEmpty()) { + parts.addObject().putObject("functionResponse").put("name", "unknown"); + } + return content; + } + + /** Gemini requires {@code functionResponse.response} to be a JSON object. */ + private ObjectNode wrapToolResponse(String responseData) { + if (responseData == null || responseData.isBlank()) { + ObjectNode empty = objectMapper.createObjectNode(); + empty.put("result", ""); + return empty; + } + try { + JsonNode parsed = objectMapper.readTree(responseData); + if (parsed.isObject()) { + return (ObjectNode) parsed; + } + ObjectNode wrapped = objectMapper.createObjectNode(); + wrapped.set("result", parsed); + return wrapped; + } catch (Exception e) { + ObjectNode wrapped = objectMapper.createObjectNode(); + wrapped.put("result", responseData); + return wrapped; + } + } + + private JsonNode parseArgs(String arguments) { + if (arguments == null || arguments.isBlank()) { + return objectMapper.createObjectNode(); + } + try { + JsonNode parsed = objectMapper.readTree(arguments); + return parsed.isObject() ? parsed : objectMapper.createObjectNode(); + } catch (Exception e) { + return objectMapper.createObjectNode(); + } + } + + private JsonNode parseSchema(String inputSchema) { + if (inputSchema == null || inputSchema.isBlank()) { + return objectMapper.createObjectNode(); + } + try { + return objectMapper.readTree(inputSchema); + } catch (Exception e) { + log.warn("[Gemini] failed to parse tool schema: {}", e.getMessage()); + return objectMapper.createObjectNode(); + } + } + + // ==================== response parsing ==================== + + /** Parse one SSE data chunk into zero or more {@link StreamEvent}s. */ + private List parseChunk(String json) { + List events = new ArrayList<>(); + try { + JsonNode node = objectMapper.readTree(json); + + JsonNode error = node.get("error"); + if (error != null && !error.isNull()) { + throw new MateClawException("err.llm.gemini_error", + "Gemini 返回错误: " + error.path("message").asText("unknown")); + } + + boolean terminal = false; + for (JsonNode candidate : node.path("candidates")) { + if (!candidate.path("finishReason").asText("").isBlank()) { + terminal = true; + } + for (JsonNode part : candidate.path("content").path("parts")) { + // Skip thinking-summary parts — they are not visible output. + if (part.path("thought").asBoolean(false)) { + continue; + } + JsonNode functionCall = part.get("functionCall"); + if (functionCall != null && !functionCall.isNull()) { + String name = functionCall.path("name").asText(""); + String id = functionCall.has("id") + ? functionCall.get("id").asText() + : "call_" + name + "_" + System.nanoTime(); + String args = functionCall.has("args") + ? functionCall.get("args").toString() : "{}"; + events.add(StreamEvent.toolCall(id, name, args)); + continue; + } + JsonNode text = part.get("text"); + if (text != null && text.isTextual() && !text.asText().isEmpty()) { + events.add(StreamEvent.text(text.asText())); + } + } + } + + // Token usage — emitted once, on the terminal chunk only. Gemini may + // repeat cumulative usageMetadata on every chunk; emitting per-chunk + // would double-count tokens in the agent's usage ledger. + JsonNode usage = node.get("usageMetadata"); + if (terminal && usage != null && !usage.isNull()) { + Integer in = usage.has("promptTokenCount") ? usage.get("promptTokenCount").asInt() : null; + Integer out = usage.has("candidatesTokenCount") + ? usage.get("candidatesTokenCount").asInt() : null; + Integer total = usage.has("totalTokenCount") + ? usage.get("totalTokenCount").asInt() + : (in != null && out != null ? in + out : null); + if (in != null || out != null || total != null) { + events.add(StreamEvent.done(in, out, total)); + } + } + } catch (MateClawException e) { + throw e; + } catch (Exception e) { + log.debug("[Gemini] skipping unparseable chunk: {}", e.getMessage()); + } + return events; + } + + private String extractErrorMessage(String errorBody) { + if (errorBody == null || errorBody.isBlank()) { + return "(empty body)"; + } + try { + JsonNode node = objectMapper.readTree(errorBody); + JsonNode message = node.path("error").path("message"); + if (message.isTextual()) { + return message.asText(); + } + } catch (Exception ignored) { + // fall through to raw body + } + return errorBody.length() > 300 ? errorBody.substring(0, 300) + "..." : errorBody; + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/llm/gemini/GeminiSchemaSanitizer.java b/mateclaw-server/src/main/java/vip/mate/llm/gemini/GeminiSchemaSanitizer.java new file mode 100644 index 00000000..c405df53 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/llm/gemini/GeminiSchemaSanitizer.java @@ -0,0 +1,111 @@ +package vip.mate.llm.gemini; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; + +import java.util.Set; + +/** + * Translates OpenAI-flavored JSON Schema tool parameters into the restricted + * {@code Schema} subset accepted by Gemini's {@code functionDeclarations.parameters}. + * + *

Tool schemas produced by Spring AI carry JSON Schema keywords that the + * Gemini API rejects ({@code $schema}, {@code additionalProperties}, + * {@code $ref}, {@code definitions}, …). This sanitizer keeps only the + * documented Gemini subset and recurses into {@code properties}, {@code items} + * and {@code anyOf}. + */ +public final class GeminiSchemaSanitizer { + + /** Keywords Gemini's {@code Schema} object understands; everything else is dropped. */ + private static final Set ALLOWED_KEYS = Set.of( + "type", "format", "title", "description", "nullable", "enum", + "maxItems", "minItems", "properties", "required", "minProperties", + "maxProperties", "minLength", "maxLength", "pattern", "example", + "anyOf", "propertyOrdering", "default", "items", "minimum", "maximum"); + + private GeminiSchemaSanitizer() {} + + /** + * Return a Gemini-compatible copy of a tool parameter schema. A null, + * empty, or non-object input yields a minimal {@code {"type":"object"}} + * schema so the function declaration always carries a valid parameters block. + */ + public static ObjectNode sanitizeToolParameters(JsonNode parameters, + com.fasterxml.jackson.databind.ObjectMapper mapper) { + JsonNode cleaned = sanitize(parameters, mapper); + if (cleaned == null || !cleaned.isObject() || cleaned.isEmpty()) { + ObjectNode fallback = mapper.createObjectNode(); + fallback.put("type", "object"); + fallback.set("properties", mapper.createObjectNode()); + return fallback; + } + return (ObjectNode) cleaned; + } + + /** Recursively strip non-Gemini keywords from an arbitrary schema node. */ + static JsonNode sanitize(JsonNode schema, com.fasterxml.jackson.databind.ObjectMapper mapper) { + if (schema == null || !schema.isObject()) { + return null; + } + ObjectNode cleaned = mapper.createObjectNode(); + schema.fields().forEachRemaining(entry -> { + String key = entry.getKey(); + JsonNode value = entry.getValue(); + if (!ALLOWED_KEYS.contains(key)) { + return; + } + switch (key) { + case "properties" -> { + if (value.isObject()) { + ObjectNode props = mapper.createObjectNode(); + value.fields().forEachRemaining(p -> { + JsonNode sub = sanitize(p.getValue(), mapper); + props.set(p.getKey(), sub != null ? sub : mapper.createObjectNode()); + }); + cleaned.set("properties", props); + } + } + case "items" -> { + JsonNode sub = sanitize(value, mapper); + cleaned.set("items", sub != null ? sub : mapper.createObjectNode()); + } + case "anyOf" -> { + if (value.isArray()) { + ArrayNode arr = mapper.createArrayNode(); + for (JsonNode item : value) { + JsonNode sub = sanitize(item, mapper); + if (sub != null) { + arr.add(sub); + } + } + cleaned.set("anyOf", arr); + } + } + default -> cleaned.set(key, value); + } + }); + + // Gemini requires every enum entry to be a string. When the parent type + // is integer/number/boolean and the enum carries non-string literals, + // drop the enum — the type plus description still guides the model and + // the tool handler validates the value anyway. + JsonNode enumVal = cleaned.get("enum"); + JsonNode typeVal = cleaned.get("type"); + if (enumVal != null && enumVal.isArray() && typeVal != null + && Set.of("integer", "number", "boolean").contains(typeVal.asText(""))) { + boolean hasNonString = false; + for (JsonNode item : enumVal) { + if (!item.isTextual()) { + hasNonString = true; + break; + } + } + if (hasNonString) { + cleaned.remove("enum"); + } + } + return cleaned; + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/tool/image/provider/GoogleImagenProvider.java b/mateclaw-server/src/main/java/vip/mate/tool/image/provider/GoogleImagenProvider.java index eef4bf1f..f5f2c42f 100644 --- a/mateclaw-server/src/main/java/vip/mate/tool/image/provider/GoogleImagenProvider.java +++ b/mateclaw-server/src/main/java/vip/mate/tool/image/provider/GoogleImagenProvider.java @@ -19,12 +19,19 @@ import java.util.List; import java.util.Set; /** - * Google Imagen 图片生成 Provider — 使用 Gemini API 的图片生成能力 - *

- * 同步模式:直接返回 Base64 图片数据。 - * 复用已有的 Google/Gemini LLM provider 的 API Key。 - *

- * API: POST https://generativelanguage.googleapis.com/v1beta/models/{model}:generateContent + * Google Gemini native image provider — "Nano Banana". + * + *

Calls the Gemini {@code generateContent} endpoint with + * {@code responseModalities:[TEXT,IMAGE]} and returns the inline base64 image + * as a {@code data:} URI. Supports both text-to-image and image editing / + * image-to-image: reference images from {@link ImageGenerationRequest#getInputImages()} + * are sent as {@code inlineData} parts alongside the prompt. + * + *

Default model is Nano Banana Pro ({@code gemini-3-pro-image-preview}); the + * original Nano Banana ({@code gemini-2.5-flash-image}) is also available. + * Reuses the {@code gemini} LLM provider's API key — no separate credential. + * + *

API: POST https://generativelanguage.googleapis.com/v1beta/models/{model}:generateContent * * @author MateClaw Team */ @@ -37,7 +44,10 @@ public class GoogleImagenProvider implements ImageGenerationProvider { private final ObjectMapper objectMapper; private static final String BASE_URL = "https://generativelanguage.googleapis.com"; - private static final String DEFAULT_MODEL = "gemini-2.0-flash-preview-image-generation"; + /** Nano Banana Pro — Gemini 3 Pro image generation. */ + private static final String DEFAULT_MODEL = "gemini-3-pro-image-preview"; + /** LLM provider id whose API key this image provider reuses. */ + private static final String LLM_PROVIDER_ID = "gemini"; @Override public String id() { @@ -46,7 +56,7 @@ public class GoogleImagenProvider implements ImageGenerationProvider { @Override public String label() { - return "Google Imagen"; + return "Google Gemini Image (Nano Banana)"; } @Override @@ -61,7 +71,7 @@ public class GoogleImagenProvider implements ImageGenerationProvider { @Override public Set capabilities() { - return Set.of(ImageCapability.TEXT_TO_IMAGE); + return Set.of(ImageCapability.TEXT_TO_IMAGE, ImageCapability.IMAGE_EDIT); } @Override @@ -69,17 +79,17 @@ public class GoogleImagenProvider implements ImageGenerationProvider { return ImageProviderCapabilities.builder() .modes(capabilities()) .supportedSizes(List.of("1024x1024", "1024x1536", "1536x1024", "1024x1792", "1792x1024")) - .aspectRatios(List.of("1:1", "3:4", "4:3", "9:16", "16:9")) - .maxCount(4) + .aspectRatios(List.of("1:1", "2:3", "3:2", "3:4", "4:3", "9:16", "16:9")) + .maxCount(1) .defaultModel(DEFAULT_MODEL) - .models(List.of("gemini-2.0-flash-preview-image-generation", "imagen-4.0-generate-preview", "imagen-4.0-ultra-generate-preview")) + .models(List.of("gemini-3-pro-image-preview", "gemini-2.5-flash-image")) .build(); } @Override public boolean isAvailable(SystemSettingsDTO config) { try { - return modelProviderService.isProviderConfigured("google"); + return modelProviderService.isProviderConfigured(LLM_PROVIDER_ID); } catch (Exception e) { return false; } @@ -90,21 +100,36 @@ public class GoogleImagenProvider implements ImageGenerationProvider { try { String apiKey = getApiKey(); if (apiKey == null) { - return ImageSubmitResult.failure(id(), "Google API Key 未配置"); + return ImageSubmitResult.failure(id(), "Gemini API Key 未配置"); } String model = request.getModel() != null && !request.getModel().isBlank() ? request.getModel() : DEFAULT_MODEL; - // 构建请求体 ObjectNode body = objectMapper.createObjectNode(); - // contents + // contents — one user turn holding the prompt text plus any reference images. ArrayNode contents = body.putArray("contents"); ObjectNode content = contents.addObject(); content.put("role", "user"); ArrayNode parts = content.putArray("parts"); - parts.addObject().put("text", request.getPrompt()); + + if (request.getPrompt() != null && !request.getPrompt().isBlank()) { + parts.addObject().put("text", request.getPrompt()); + } + // Reference images (image edit / image-to-image): inline as base64 parts. + List inputImages = request.getInputImages(); + boolean editing = inputImages != null && !inputImages.isEmpty(); + if (inputImages != null) { + for (ImageReference ref : inputImages) { + if (ref == null || ref.data() == null || ref.data().length == 0) { + continue; + } + ObjectNode inlineData = parts.addObject().putObject("inlineData"); + inlineData.put("mimeType", ref.mimeType() != null ? ref.mimeType() : "image/png"); + inlineData.put("data", Base64.getEncoder().encodeToString(ref.data())); + } + } // generationConfig ObjectNode genConfig = body.putObject("generationConfig"); @@ -112,43 +137,54 @@ public class GoogleImagenProvider implements ImageGenerationProvider { modalities.add("TEXT"); modalities.add("IMAGE"); - if (request.getAspectRatio() != null) { - ObjectNode imageConfig = genConfig.putObject("imageConfig"); + ObjectNode imageConfig = objectMapper.createObjectNode(); + if (request.getAspectRatio() != null && !request.getAspectRatio().isBlank()) { imageConfig.put("aspectRatio", request.getAspectRatio()); } + // Nano Banana Pro resolution tier (1K / 2K / 4K) — opt-in via extraParams. + Object imageSize = request.getExtraParams() != null + ? request.getExtraParams().get("imageSize") : null; + if (imageSize instanceof String sizeTier && !sizeTier.isBlank()) { + imageConfig.put("imageSize", sizeTier); + } + if (!imageConfig.isEmpty()) { + genConfig.set("imageConfig", imageConfig); + } String url = BASE_URL + "/v1beta/models/" + model + ":generateContent?key=" + apiKey; HttpResponse response = HttpRequest.post(url) .header("Content-Type", "application/json") .body(body.toString()) - .timeout(60_000) + .timeout(120_000) .execute(); if (response.getStatus() != 200) { String errBody = response.body(); - log.warn("[Google Imagen] Failed: HTTP {} - {}", response.getStatus(), errBody); - return ImageSubmitResult.failure(id(), "Google Imagen 失败: HTTP " + response.getStatus()); + log.warn("[Nano Banana] Failed: HTTP {} - {}", response.getStatus(), errBody); + return ImageSubmitResult.failure(id(), "Gemini 图像生成失败: HTTP " + response.getStatus()); } JsonNode result = objectMapper.readTree(response.body()); List imageUrls = extractImagesFromResponse(result); if (imageUrls.isEmpty()) { - return ImageSubmitResult.failure(id(), "Google Imagen 未返回图片"); + return ImageSubmitResult.failure(id(), "Gemini 未返回图片"); } - log.info("[Google Imagen] Generated {} images (model={})", imageUrls.size(), model); + log.info("[Nano Banana] Generated {} image(s) (model={}, editing={})", + imageUrls.size(), model, editing); return ImageSubmitResult.syncSuccess(id(), imageUrls); } catch (Exception e) { - log.error("[Google Imagen] Error: {}", e.getMessage(), e); - return ImageSubmitResult.failure(id(), "Google Imagen 异常: " + e.getMessage()); + log.error("[Nano Banana] Error: {}", e.getMessage(), e); + return ImageSubmitResult.failure(id(), "Gemini 图像生成异常: " + e.getMessage()); } } /** - * 从 Gemini 响应中提取 Base64 图片,转换为 data URI + * Extract base64 images from a Gemini generateContent response, converting + * each {@code inlineData} part to a {@code data:} URI. */ private List extractImagesFromResponse(JsonNode result) { List images = new ArrayList<>(); @@ -159,7 +195,7 @@ public class GoogleImagenProvider implements ImageGenerationProvider { JsonNode parts = candidate.path("content").path("parts"); if (parts.isArray()) { for (JsonNode part : parts) { - // 尝试 inlineData 或 inline_data + // Accept both inlineData (camelCase) and inline_data (snake_case). JsonNode inlineData = part.has("inlineData") ? part.get("inlineData") : part.path("inline_data"); if (inlineData.has("data")) { @@ -167,7 +203,6 @@ public class GoogleImagenProvider implements ImageGenerationProvider { ? inlineData.get("mimeType").asText("image/png") : inlineData.path("mime_type").asText("image/png"); String base64Data = inlineData.get("data").asText(); - // 返回 data URI 格式 images.add("data:" + mimeType + ";base64," + base64Data); } } @@ -179,7 +214,7 @@ public class GoogleImagenProvider implements ImageGenerationProvider { private String getApiKey() { try { - return modelProviderService.getProviderConfig("google").getApiKey(); + return modelProviderService.getProviderConfig(LLM_PROVIDER_ID).getApiKey(); } catch (Exception e) { return null; } diff --git a/mateclaw-server/src/main/resources/db/data-en.sql b/mateclaw-server/src/main/resources/db/data-en.sql index 3564ff6f..ecee62e0 100644 --- a/mateclaw-server/src/main/resources/db/data-en.sql +++ b/mateclaw-server/src/main/resources/db/data-en.sql @@ -109,6 +109,10 @@ MERGE INTO mate_model_provider (provider_id, name, api_key_prefix, chat_model, a KEY (provider_id) VALUES ('gemini', 'Google Gemini', '', 'GeminiChatModel', '', 'https://generativelanguage.googleapis.com', '{}', FALSE, FALSE, TRUE, TRUE, TRUE, TRUE, NOW(), NOW()); +MERGE INTO mate_model_provider (provider_id, name, api_key_prefix, chat_model, api_key, base_url, generate_kwargs, is_custom, is_local, support_model_discovery, support_connection_check, freeze_url, require_api_key, create_time, update_time) +KEY (provider_id) +VALUES ('xai', 'xAI (Grok)', 'xai-', 'OpenAIChatModel', '', 'https://api.x.ai/v1', '{}', FALSE, FALSE, TRUE, TRUE, TRUE, TRUE, NOW(), NOW()); + MERGE INTO mate_model_provider (provider_id, name, api_key_prefix, chat_model, api_key, base_url, generate_kwargs, is_custom, is_local, support_model_discovery, support_connection_check, freeze_url, require_api_key, create_time, update_time) KEY (provider_id) VALUES ('openrouter', 'OpenRouter', 'sk-or-', 'OpenAIChatModel', '', 'https://openrouter.ai/api/v1', '{}', FALSE, FALSE, TRUE, TRUE, TRUE, TRUE, NOW(), NOW()); @@ -275,6 +279,10 @@ MERGE INTO mate_model_config (id, name, provider, model_name, description, tempe (1000000159, 'Gemini 2.5 Flash', 'gemini', 'gemini-2.5-flash', '', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0), (1000000160, 'Gemini 2.5 Flash Lite', 'gemini', 'gemini-2.5-flash-lite', '', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0), (1000000161, 'Gemini 2.0 Flash', 'gemini', 'gemini-2.0-flash', '', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0), +(1000000340, 'Grok 4', 'xai', 'grok-4', '', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0), +(1000000341, 'Grok 4 Fast', 'xai', 'grok-4-fast', '', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0), +(1000000342, 'Grok 3', 'xai', 'grok-3', '', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0), +(1000000343, 'Grok 3 Mini', 'xai', 'grok-3-mini', '', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0), (1000000200, 'GPT-5', 'openrouter', 'openai/gpt-5', 'GPT-5 via OpenRouter', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0), (1000000201, 'Claude Opus 4.6', 'openrouter', 'anthropic/claude-opus-4-6', 'Claude Opus 4.6 via OpenRouter', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0), (1000000202, 'Claude Sonnet 4.6', 'openrouter', 'anthropic/claude-sonnet-4-6', 'Claude Sonnet 4.6 via OpenRouter', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0), diff --git a/mateclaw-server/src/main/resources/db/data-mysql-en.sql b/mateclaw-server/src/main/resources/db/data-mysql-en.sql index e5183ead..2d197d95 100644 --- a/mateclaw-server/src/main/resources/db/data-mysql-en.sql +++ b/mateclaw-server/src/main/resources/db/data-mysql-en.sql @@ -114,6 +114,10 @@ INSERT INTO mate_model_provider (provider_id, name, api_key_prefix, chat_model, VALUES ('gemini', 'Google Gemini', '', 'GeminiChatModel', '', 'https://generativelanguage.googleapis.com', '{}', FALSE, FALSE, TRUE, TRUE, TRUE, TRUE, NOW(), NOW()) ON DUPLICATE KEY UPDATE name=VALUES(name), api_key_prefix=VALUES(api_key_prefix), chat_model=VALUES(chat_model), api_key=VALUES(api_key), base_url=VALUES(base_url), generate_kwargs=VALUES(generate_kwargs), is_custom=VALUES(is_custom), is_local=VALUES(is_local), support_model_discovery=VALUES(support_model_discovery), support_connection_check=VALUES(support_connection_check), freeze_url=VALUES(freeze_url), require_api_key=VALUES(require_api_key), update_time=VALUES(update_time); +INSERT INTO mate_model_provider (provider_id, name, api_key_prefix, chat_model, api_key, base_url, generate_kwargs, is_custom, is_local, support_model_discovery, support_connection_check, freeze_url, require_api_key, create_time, update_time) +VALUES ('xai', 'xAI (Grok)', 'xai-', 'OpenAIChatModel', '', 'https://api.x.ai/v1', '{}', FALSE, FALSE, TRUE, TRUE, TRUE, TRUE, NOW(), NOW()) +ON DUPLICATE KEY UPDATE name=VALUES(name), api_key_prefix=VALUES(api_key_prefix), chat_model=VALUES(chat_model), api_key=VALUES(api_key), base_url=VALUES(base_url), generate_kwargs=VALUES(generate_kwargs), is_custom=VALUES(is_custom), is_local=VALUES(is_local), support_model_discovery=VALUES(support_model_discovery), support_connection_check=VALUES(support_connection_check), freeze_url=VALUES(freeze_url), require_api_key=VALUES(require_api_key), update_time=VALUES(update_time); + INSERT INTO mate_model_provider (provider_id, name, api_key_prefix, chat_model, api_key, base_url, generate_kwargs, is_custom, is_local, support_model_discovery, support_connection_check, freeze_url, require_api_key, create_time, update_time) VALUES ('openrouter', 'OpenRouter', 'sk-or-', 'OpenAIChatModel', '', 'https://openrouter.ai/api/v1', '{}', FALSE, FALSE, TRUE, TRUE, TRUE, TRUE, NOW(), NOW()) ON DUPLICATE KEY UPDATE name=VALUES(name), api_key_prefix=VALUES(api_key_prefix), chat_model=VALUES(chat_model), api_key=VALUES(api_key), base_url=VALUES(base_url), generate_kwargs=VALUES(generate_kwargs), is_custom=VALUES(is_custom), is_local=VALUES(is_local), support_model_discovery=VALUES(support_model_discovery), support_connection_check=VALUES(support_connection_check), freeze_url=VALUES(freeze_url), require_api_key=VALUES(require_api_key), update_time=VALUES(update_time); @@ -308,6 +312,10 @@ VALUES (1000000159, 'Gemini 2.5 Flash', 'gemini', 'gemini-2.5-flash', '', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0), (1000000160, 'Gemini 2.5 Flash Lite', 'gemini', 'gemini-2.5-flash-lite', '', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0), (1000000161, 'Gemini 2.0 Flash', 'gemini', 'gemini-2.0-flash', '', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0), +(1000000340, 'Grok 4', 'xai', 'grok-4', '', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0), +(1000000341, 'Grok 4 Fast', 'xai', 'grok-4-fast', '', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0), +(1000000342, 'Grok 3', 'xai', 'grok-3', '', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0), +(1000000343, 'Grok 3 Mini', 'xai', 'grok-3-mini', '', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0), (1000000200, 'GPT-5', 'openrouter', 'openai/gpt-5', 'GPT-5 via OpenRouter', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0), (1000000201, 'Claude Opus 4.6', 'openrouter', 'anthropic/claude-opus-4-6', 'Claude Opus 4.6 via OpenRouter', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0), (1000000202, 'Claude Sonnet 4.6', 'openrouter', 'anthropic/claude-sonnet-4-6', 'Claude Sonnet 4.6 via OpenRouter', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0), diff --git a/mateclaw-server/src/main/resources/db/data-mysql-zh.sql b/mateclaw-server/src/main/resources/db/data-mysql-zh.sql index fe3787a8..b778b395 100644 --- a/mateclaw-server/src/main/resources/db/data-mysql-zh.sql +++ b/mateclaw-server/src/main/resources/db/data-mysql-zh.sql @@ -113,6 +113,10 @@ INSERT INTO mate_model_provider (provider_id, name, api_key_prefix, chat_model, VALUES ('gemini', 'Google Gemini', '', 'GeminiChatModel', '', 'https://generativelanguage.googleapis.com', '{}', FALSE, FALSE, TRUE, TRUE, TRUE, TRUE, NOW(), NOW()) ON DUPLICATE KEY UPDATE name=VALUES(name), api_key_prefix=VALUES(api_key_prefix), chat_model=VALUES(chat_model), api_key=VALUES(api_key), base_url=VALUES(base_url), generate_kwargs=VALUES(generate_kwargs), is_custom=VALUES(is_custom), is_local=VALUES(is_local), support_model_discovery=VALUES(support_model_discovery), support_connection_check=VALUES(support_connection_check), freeze_url=VALUES(freeze_url), require_api_key=VALUES(require_api_key), update_time=VALUES(update_time); +INSERT INTO mate_model_provider (provider_id, name, api_key_prefix, chat_model, api_key, base_url, generate_kwargs, is_custom, is_local, support_model_discovery, support_connection_check, freeze_url, require_api_key, create_time, update_time) +VALUES ('xai', 'xAI (Grok)', 'xai-', 'OpenAIChatModel', '', 'https://api.x.ai/v1', '{}', FALSE, FALSE, TRUE, TRUE, TRUE, TRUE, NOW(), NOW()) +ON DUPLICATE KEY UPDATE name=VALUES(name), api_key_prefix=VALUES(api_key_prefix), chat_model=VALUES(chat_model), api_key=VALUES(api_key), base_url=VALUES(base_url), generate_kwargs=VALUES(generate_kwargs), is_custom=VALUES(is_custom), is_local=VALUES(is_local), support_model_discovery=VALUES(support_model_discovery), support_connection_check=VALUES(support_connection_check), freeze_url=VALUES(freeze_url), require_api_key=VALUES(require_api_key), update_time=VALUES(update_time); + INSERT INTO mate_model_provider (provider_id, name, api_key_prefix, chat_model, api_key, base_url, generate_kwargs, is_custom, is_local, support_model_discovery, support_connection_check, freeze_url, require_api_key, create_time, update_time) VALUES ('openrouter', 'OpenRouter', 'sk-or-', 'OpenAIChatModel', '', 'https://openrouter.ai/api/v1', '{}', FALSE, FALSE, TRUE, TRUE, TRUE, TRUE, NOW(), NOW()) ON DUPLICATE KEY UPDATE name=VALUES(name), api_key_prefix=VALUES(api_key_prefix), chat_model=VALUES(chat_model), api_key=VALUES(api_key), base_url=VALUES(base_url), generate_kwargs=VALUES(generate_kwargs), is_custom=VALUES(is_custom), is_local=VALUES(is_local), support_model_discovery=VALUES(support_model_discovery), support_connection_check=VALUES(support_connection_check), freeze_url=VALUES(freeze_url), require_api_key=VALUES(require_api_key), update_time=VALUES(update_time); @@ -305,6 +309,10 @@ VALUES (1000000159, 'Gemini 2.5 Flash', 'gemini', 'gemini-2.5-flash', '', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0), (1000000160, 'Gemini 2.5 Flash Lite', 'gemini', 'gemini-2.5-flash-lite', '', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0), (1000000161, 'Gemini 2.0 Flash', 'gemini', 'gemini-2.0-flash', '', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0), +(1000000340, 'Grok 4', 'xai', 'grok-4', '', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0), +(1000000341, 'Grok 4 Fast', 'xai', 'grok-4-fast', '', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0), +(1000000342, 'Grok 3', 'xai', 'grok-3', '', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0), +(1000000343, 'Grok 3 Mini', 'xai', 'grok-3-mini', '', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0), (1000000200, 'GPT-5', 'openrouter', 'openai/gpt-5', 'OpenRouter 代理 GPT-5', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0), (1000000201, 'Claude Opus 4.6', 'openrouter', 'anthropic/claude-opus-4-6', 'OpenRouter 代理 Claude Opus 4.6', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0), (1000000202, 'Claude Sonnet 4.6', 'openrouter', 'anthropic/claude-sonnet-4-6', 'OpenRouter 代理 Claude Sonnet 4.6', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0), diff --git a/mateclaw-server/src/main/resources/db/data-zh.sql b/mateclaw-server/src/main/resources/db/data-zh.sql index f0060263..a663950d 100644 --- a/mateclaw-server/src/main/resources/db/data-zh.sql +++ b/mateclaw-server/src/main/resources/db/data-zh.sql @@ -109,6 +109,10 @@ MERGE INTO mate_model_provider (provider_id, name, api_key_prefix, chat_model, a KEY (provider_id) VALUES ('gemini', 'Google Gemini', '', 'GeminiChatModel', '', 'https://generativelanguage.googleapis.com', '{}', FALSE, FALSE, TRUE, TRUE, TRUE, TRUE, NOW(), NOW()); +MERGE INTO mate_model_provider (provider_id, name, api_key_prefix, chat_model, api_key, base_url, generate_kwargs, is_custom, is_local, support_model_discovery, support_connection_check, freeze_url, require_api_key, create_time, update_time) +KEY (provider_id) +VALUES ('xai', 'xAI (Grok)', 'xai-', 'OpenAIChatModel', '', 'https://api.x.ai/v1', '{}', FALSE, FALSE, TRUE, TRUE, TRUE, TRUE, NOW(), NOW()); + MERGE INTO mate_model_provider (provider_id, name, api_key_prefix, chat_model, api_key, base_url, generate_kwargs, is_custom, is_local, support_model_discovery, support_connection_check, freeze_url, require_api_key, create_time, update_time) KEY (provider_id) VALUES ('openrouter', 'OpenRouter', 'sk-or-', 'OpenAIChatModel', '', 'https://openrouter.ai/api/v1', '{}', FALSE, FALSE, TRUE, TRUE, TRUE, TRUE, NOW(), NOW()); @@ -277,6 +281,10 @@ MERGE INTO mate_model_config (id, name, provider, model_name, description, tempe (1000000159, 'Gemini 2.5 Flash', 'gemini', 'gemini-2.5-flash', '', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0), (1000000160, 'Gemini 2.5 Flash Lite', 'gemini', 'gemini-2.5-flash-lite', '', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0), (1000000161, 'Gemini 2.0 Flash', 'gemini', 'gemini-2.0-flash', '', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0), +(1000000340, 'Grok 4', 'xai', 'grok-4', '', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0), +(1000000341, 'Grok 4 Fast', 'xai', 'grok-4-fast', '', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0), +(1000000342, 'Grok 3', 'xai', 'grok-3', '', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0), +(1000000343, 'Grok 3 Mini', 'xai', 'grok-3-mini', '', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0), (1000000200, 'GPT-5', 'openrouter', 'openai/gpt-5', 'OpenRouter 代理 GPT-5', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0), (1000000201, 'Claude Opus 4.6', 'openrouter', 'anthropic/claude-opus-4-6', 'OpenRouter 代理 Claude Opus 4.6', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0), (1000000202, 'Claude Sonnet 4.6', 'openrouter', 'anthropic/claude-sonnet-4-6', 'OpenRouter 代理 Claude Sonnet 4.6', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0), diff --git a/mateclaw-server/src/main/resources/db/migration/h2/V115__xai_grok_provider.sql b/mateclaw-server/src/main/resources/db/migration/h2/V115__xai_grok_provider.sql new file mode 100644 index 00000000..2fa8dd76 --- /dev/null +++ b/mateclaw-server/src/main/resources/db/migration/h2/V115__xai_grok_provider.sql @@ -0,0 +1,35 @@ +-- V115: register the xAI (Grok) provider plus its Grok 3 / Grok 4 model catalog. +-- +-- xAI's API is OpenAI-compatible (https://api.x.ai/v1), so the provider runs on +-- OpenAIChatModel — no new protocol or ChatModelBuilder is required; +-- OpenAiCompatibleChatModelBuilder + OpenAiCompatibleListModelsProbe handle it. +-- +-- Existing seed files db/data-*.sql already carry the same rows for fresh +-- installs; this migration is the upgrade path for already-deployed databases +-- (DatabaseBootstrapRunner skips the seed when mate_user is non-empty). + +-- -- Provider -------------------------------------------------------------- +MERGE INTO mate_model_provider (provider_id, name, api_key_prefix, chat_model, api_key, base_url, generate_kwargs, is_custom, is_local, support_model_discovery, support_connection_check, freeze_url, require_api_key, create_time, update_time) +KEY (provider_id) +VALUES ( + 'xai', + 'xAI (Grok)', + 'xai-', + 'OpenAIChatModel', + '', + 'https://api.x.ai/v1', + '{}', + FALSE, FALSE, TRUE, TRUE, TRUE, TRUE, + NOW(), NOW() +); + +-- -- Model catalog --------------------------------------------------------- +-- IDs use the 1000000340-1000000343 block reserved for xAI so future Grok +-- additions can grow contiguously. +MERGE INTO mate_model_config (id, name, provider, model_name, description, temperature, max_tokens, top_p, builtin, enabled, is_default, create_time, update_time, deleted) +KEY (id) +VALUES + (1000000340, 'Grok 4', 'xai', 'grok-4', '', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0), + (1000000341, 'Grok 4 Fast', 'xai', 'grok-4-fast', '', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0), + (1000000342, 'Grok 3', 'xai', 'grok-3', '', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0), + (1000000343, 'Grok 3 Mini', 'xai', 'grok-3-mini', '', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0); diff --git a/mateclaw-server/src/main/resources/db/migration/mysql/V115__xai_grok_provider.sql b/mateclaw-server/src/main/resources/db/migration/mysql/V115__xai_grok_provider.sql new file mode 100644 index 00000000..db1dbd05 --- /dev/null +++ b/mateclaw-server/src/main/resources/db/migration/mysql/V115__xai_grok_provider.sql @@ -0,0 +1,47 @@ +-- V115: register the xAI (Grok) provider plus its Grok 3 / Grok 4 model catalog. +-- +-- See the H2 copy for full background. The MySQL copy uses INSERT ... ON +-- DUPLICATE KEY UPDATE; the api_key column is intentionally omitted from the +-- update list so existing deployments that have already configured a key keep it. + +-- -- Provider -------------------------------------------------------------- +INSERT INTO mate_model_provider (provider_id, name, api_key_prefix, chat_model, api_key, base_url, generate_kwargs, is_custom, is_local, support_model_discovery, support_connection_check, freeze_url, require_api_key, create_time, update_time) +VALUES ( + 'xai', + 'xAI (Grok)', + 'xai-', + 'OpenAIChatModel', + '', + 'https://api.x.ai/v1', + '{}', + FALSE, FALSE, TRUE, TRUE, TRUE, TRUE, + NOW(), NOW() +) +ON DUPLICATE KEY UPDATE + name = VALUES(name), + api_key_prefix = VALUES(api_key_prefix), + chat_model = VALUES(chat_model), + base_url = VALUES(base_url), + generate_kwargs = VALUES(generate_kwargs), + support_model_discovery = VALUES(support_model_discovery), + support_connection_check = VALUES(support_connection_check), + freeze_url = VALUES(freeze_url), + require_api_key = VALUES(require_api_key), + update_time = VALUES(update_time); + +-- -- Model catalog --------------------------------------------------------- +-- IDs use the 1000000340-1000000343 block reserved for xAI so future Grok +-- additions can grow contiguously. +INSERT INTO mate_model_config (id, name, provider, model_name, description, temperature, max_tokens, top_p, builtin, enabled, is_default, create_time, update_time, deleted) +VALUES + (1000000340, 'Grok 4', 'xai', 'grok-4', '', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0), + (1000000341, 'Grok 4 Fast', 'xai', 'grok-4-fast', '', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0), + (1000000342, 'Grok 3', 'xai', 'grok-3', '', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0), + (1000000343, 'Grok 3 Mini', 'xai', 'grok-3-mini', '', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0) +ON DUPLICATE KEY UPDATE + name = VALUES(name), + model_name = VALUES(model_name), + description = VALUES(description), + builtin = VALUES(builtin), + enabled = VALUES(enabled), + update_time = VALUES(update_time); diff --git a/mateclaw-server/src/main/resources/messages.properties b/mateclaw-server/src/main/resources/messages.properties index f9213a24..4c8c054b 100644 --- a/mateclaw-server/src/main/resources/messages.properties +++ b/mateclaw-server/src/main/resources/messages.properties @@ -205,13 +205,15 @@ err.agent.model_not_configured=\u6a21\u578b Provider \u672a\u5b8c\u6210\u914d\u7 err.agent.protocol_not_supported=\u5f53\u524d\u4e0d\u652f\u6301\u8be5\u534f\u8bae err.agent.plan_compile_failed=Plan-Execute StateGraph \u7f16\u8bd1\u5931\u8d25 err.agent.graph_compile_failed=StateGraph v2 \u7f16\u8bd1\u5931\u8d25 -err.agent.protocol_limited=StateGraph \u5f53\u524d\u4ec5\u652f\u6301 DashScope/OpenAI/Anthropic \u534f\u8bae +err.agent.protocol_limited=StateGraph \u5f53\u524d\u4ec5\u652f\u6301 DashScope/OpenAI/Anthropic/Gemini \u534f\u8bae err.agent.provider_not_configured=Provider \u672a\u5b8c\u6210\u914d\u7f6e err.agent.provider_apikey_invalid=Provider API Key \u672a\u914d\u7f6e\u6216\u65e0\u6548 err.agent.provider_baseurl_missing=Provider Base URL \u672a\u914d\u7f6e err.agent.dashscope_key_missing=DashScope API Key \u672a\u914d\u7f6e err.agent.anthropic_not_configured=Anthropic Provider \u672a\u5b8c\u6210\u914d\u7f6e err.agent.anthropic_key_invalid=Anthropic API Key \u672a\u914d\u7f6e\u6216\u65e0\u6548 +err.agent.gemini_not_configured=Gemini Provider \u672a\u5b8c\u6210\u914d\u7f6e +err.agent.gemini_key_invalid=Gemini API Key \u672a\u914d\u7f6e\u6216\u65e0\u6548 err.agent.template_not_found=\u6a21\u677f\u4e0d\u5b58\u5728 err.agent.delete_forbidden=\u53ea\u6709\u521b\u5efa\u8005\u6216\u5de5\u4f5c\u533a\u7ba1\u7406\u5458\u53ef\u5220\u9664\u6b64 Agent err.common.wrong_workspace=\u8d44\u6e90\u4e0d\u5c5e\u4e8e\u5f53\u524d\u5de5\u4f5c\u533a @@ -257,6 +259,8 @@ err.llm.chatgpt_models_fetch_failed=\u62c9\u53d6 ChatGPT \u53ef\u7528\u6a21\u578 err.llm.chatgpt_stream_failed=ChatGPT \u6d41\u5f0f\u8c03\u7528\u5931\u8d25 err.llm.chatgpt_error=ChatGPT \u8fd4\u56de\u9519\u8bef err.llm.chatgpt_account_missing=chatgpt-account-id \u7f3a\u5931 +err.llm.gemini_stream_failed=Gemini \u6d41\u5f0f\u8c03\u7528\u5931\u8d25 +err.llm.gemini_error=Gemini \u8fd4\u56de\u9519\u8bef err.llm.model_not_supported=\u6a21\u578b ID \u4e0d\u652f\u6301\u5728 DashScope \u539f\u751f\u534f\u8bae\u4e2d\u4f7f\u7528\u3002\u70b9\u7248\u672c\u683c\u5f0f\u7684\u7cfb\u5217\uff08\u4f8b\u5982 qwen3.5-*\u3001qwen3.6-*\uff09\u53ea\u80fd\u901a\u8fc7\u517c\u5bb9\u6a21\u5f0f\u4f7f\u7528\u3002\u8bf7\u4f7f\u7528\u5141\u8bb8\u7684 ID\uff0c\u5982 qwen-max / qwen-plus / qwen3-max\u3002 # datasource err.datasource.not_found=\u6570\u636e\u6e90\u4e0d\u5b58\u5728 diff --git a/mateclaw-server/src/main/resources/messages_en.properties b/mateclaw-server/src/main/resources/messages_en.properties index ab202472..74a77a3d 100644 --- a/mateclaw-server/src/main/resources/messages_en.properties +++ b/mateclaw-server/src/main/resources/messages_en.properties @@ -217,13 +217,15 @@ err.agent.model_not_configured=Model provider not configured, please fill in API err.agent.protocol_not_supported=Protocol not currently supported err.agent.plan_compile_failed=Plan-Execute StateGraph compilation failed err.agent.graph_compile_failed=StateGraph v2 compilation failed -err.agent.protocol_limited=StateGraph currently only supports DashScope, OpenAI-compatible, and Anthropic protocols +err.agent.protocol_limited=StateGraph currently only supports DashScope, OpenAI-compatible, Anthropic, and Gemini protocols err.agent.provider_not_configured=Provider not configured, please fill in valid API Key and Base URL err.agent.provider_apikey_invalid=Provider API Key not configured or invalid err.agent.provider_baseurl_missing=Provider Base URL not configured err.agent.dashscope_key_missing=DashScope API Key not configured err.agent.anthropic_not_configured=Anthropic Provider not configured err.agent.anthropic_key_invalid=Anthropic API Key not configured or invalid +err.agent.gemini_not_configured=Gemini Provider not configured +err.agent.gemini_key_invalid=Gemini API Key not configured or invalid err.agent.template_not_found=Template not found err.agent.delete_forbidden=Only the creator or a workspace admin can delete this Agent err.common.wrong_workspace=Resource does not belong to current workspace @@ -269,6 +271,8 @@ err.llm.chatgpt_models_fetch_failed=Failed to fetch available ChatGPT models err.llm.chatgpt_stream_failed=ChatGPT streaming call failed err.llm.chatgpt_error=ChatGPT returned an error err.llm.chatgpt_account_missing=chatgpt-account-id missing, disconnect and re-login via OAuth +err.llm.gemini_stream_failed=Gemini streaming call failed +err.llm.gemini_error=Gemini returned an error # datasource err.datasource.not_found=Datasource not found err.datasource.sql_empty=SQL cannot be empty diff --git a/mateclaw-server/src/test/java/vip/mate/llm/gemini/GeminiNativeClientTest.java b/mateclaw-server/src/test/java/vip/mate/llm/gemini/GeminiNativeClientTest.java new file mode 100644 index 00000000..43dfe5d9 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/llm/gemini/GeminiNativeClientTest.java @@ -0,0 +1,138 @@ +package vip.mate.llm.gemini; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.springframework.ai.chat.messages.AssistantMessage; +import org.springframework.ai.chat.messages.Message; +import org.springframework.ai.chat.messages.SystemMessage; +import org.springframework.ai.chat.messages.ToolResponseMessage; +import org.springframework.ai.chat.messages.UserMessage; +import org.springframework.ai.tool.definition.ToolDefinition; +import vip.mate.llm.gemini.GeminiNativeClient.GeminiCall; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Tests for {@link GeminiNativeClient#buildRequestBody} — the Spring AI + * {@code Message} list → Gemini {@code generateContent} request translation. + */ +class GeminiNativeClientTest { + + private final ObjectMapper mapper = new ObjectMapper(); + private final GeminiNativeClient client = new GeminiNativeClient(mapper); + + private GeminiCall call(List messages, List tools) { + return new GeminiCall("https://generativelanguage.googleapis.com", "test-key", + "gemini-3-pro-preview", messages, 0.7, 4096, tools); + } + + @Test + @DisplayName("system message is hoisted into systemInstruction") + void systemMessageBecomesSystemInstruction() { + ObjectNode body = client.buildRequestBody(call( + List.of(new SystemMessage("You are helpful"), new UserMessage("Hi")), null)); + + assertEquals("You are helpful", + body.path("systemInstruction").path("parts").path(0).path("text").asText()); + // The system turn must NOT also appear in contents. + assertEquals(1, body.path("contents").size()); + assertEquals("user", body.path("contents").path(0).path("role").asText()); + } + + @Test + @DisplayName("user message maps to a user-role text part") + void userMessageMapsToUserContent() { + ObjectNode body = client.buildRequestBody(call( + List.of(new UserMessage("What is the weather?")), null)); + + JsonNode content = body.path("contents").path(0); + assertEquals("user", content.path("role").asText()); + assertEquals("What is the weather?", content.path("parts").path(0).path("text").asText()); + } + + @Test + @DisplayName("assistant tool call maps to a model-role functionCall part") + void assistantToolCallMapsToFunctionCall() { + AssistantMessage assistant = AssistantMessage.builder() + .content("") + .toolCalls(List.of(new AssistantMessage.ToolCall( + "call_1", "function", "get_weather", "{\"city\":\"NYC\"}"))) + .build(); + ObjectNode body = client.buildRequestBody(call( + List.of(new UserMessage("weather?"), assistant), null)); + + JsonNode modelContent = body.path("contents").path(1); + assertEquals("model", modelContent.path("role").asText()); + JsonNode functionCall = modelContent.path("parts").path(0).path("functionCall"); + assertEquals("get_weather", functionCall.path("name").asText()); + assertEquals("NYC", functionCall.path("args").path("city").asText()); + } + + @Test + @DisplayName("tool response maps to a user-role functionResponse with an object payload") + void toolResponseMapsToFunctionResponse() { + ToolResponseMessage toolMsg = ToolResponseMessage.builder() + .responses(List.of( + new ToolResponseMessage.ToolResponse("call_1", "get_weather", "{\"temp\":20}"))) + .build(); + ObjectNode body = client.buildRequestBody(call( + List.of(new UserMessage("weather?"), toolMsg), null)); + + JsonNode functionResponse = body.path("contents").path(1) + .path("parts").path(0).path("functionResponse"); + assertEquals("get_weather", functionResponse.path("name").asText()); + assertEquals(20, functionResponse.path("response").path("temp").asInt()); + } + + @Test + @DisplayName("non-object tool response is wrapped under a result key") + void nonObjectToolResponseIsWrapped() { + ToolResponseMessage toolMsg = ToolResponseMessage.builder() + .responses(List.of( + new ToolResponseMessage.ToolResponse("call_1", "echo", "plain text result"))) + .build(); + ObjectNode body = client.buildRequestBody(call( + List.of(new UserMessage("echo"), toolMsg), null)); + + JsonNode response = body.path("contents").path(1) + .path("parts").path(0).path("functionResponse").path("response"); + assertTrue(response.isObject()); + assertEquals("plain text result", response.path("result").asText()); + } + + @Test + @DisplayName("tool definitions become sanitized functionDeclarations") + void toolsBecomeFunctionDeclarations() { + ToolDefinition tool = ToolDefinition.builder() + .name("get_weather") + .description("Get the weather for a city") + .inputSchema("{\"$schema\":\"x\",\"type\":\"object\"," + + "\"properties\":{\"city\":{\"type\":\"string\"}}}") + .build(); + ObjectNode body = client.buildRequestBody(call( + List.of(new UserMessage("weather?")), List.of(tool))); + + JsonNode decl = body.path("tools").path(0).path("functionDeclarations").path(0); + assertEquals("get_weather", decl.path("name").asText()); + assertEquals("Get the weather for a city", decl.path("description").asText()); + assertEquals("string", decl.path("parameters").path("properties").path("city").path("type").asText()); + assertFalse(decl.path("parameters").has("$schema"), "schema must be sanitized"); + } + + @Test + @DisplayName("generationConfig carries temperature and maxOutputTokens") + void generationConfigCarriesSamplingParams() { + ObjectNode body = client.buildRequestBody(call( + List.of(new UserMessage("hi")), null)); + + assertEquals(0.7, body.path("generationConfig").path("temperature").asDouble(), 1e-9); + assertEquals(4096, body.path("generationConfig").path("maxOutputTokens").asInt()); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/llm/gemini/GeminiSchemaSanitizerTest.java b/mateclaw-server/src/test/java/vip/mate/llm/gemini/GeminiSchemaSanitizerTest.java new file mode 100644 index 00000000..1377a40a --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/llm/gemini/GeminiSchemaSanitizerTest.java @@ -0,0 +1,118 @@ +package vip.mate.llm.gemini; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Tests for {@link GeminiSchemaSanitizer} — the JSON Schema → Gemini + * {@code Schema} subset translation used when sending tool declarations. + */ +class GeminiSchemaSanitizerTest { + + private final ObjectMapper mapper = new ObjectMapper(); + + private JsonNode parse(String json) { + try { + return mapper.readTree(json); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + @Test + @DisplayName("drops unsupported JSON Schema keywords") + void dropsUnsupportedKeywords() { + JsonNode schema = parse(""" + { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": false, + "properties": { + "city": {"type": "string", "description": "City name"} + }, + "required": ["city"] + } + """); + + ObjectNode cleaned = GeminiSchemaSanitizer.sanitizeToolParameters(schema, mapper); + + assertFalse(cleaned.has("$schema"), "$schema must be stripped"); + assertFalse(cleaned.has("additionalProperties"), "additionalProperties must be stripped"); + assertEquals("object", cleaned.path("type").asText()); + assertTrue(cleaned.path("properties").has("city")); + assertEquals("City name", cleaned.path("properties").path("city").path("description").asText()); + assertTrue(cleaned.path("required").isArray()); + } + + @Test + @DisplayName("recurses into nested properties and array items") + void recursesIntoNested() { + JsonNode schema = parse(""" + { + "type": "object", + "properties": { + "tags": { + "type": "array", + "additionalProperties": true, + "items": {"type": "string", "$comment": "drop me"} + }, + "nested": { + "type": "object", + "$ref": "#/defs/x", + "properties": {"inner": {"type": "number"}} + } + } + } + """); + + ObjectNode cleaned = GeminiSchemaSanitizer.sanitizeToolParameters(schema, mapper); + + JsonNode tags = cleaned.path("properties").path("tags"); + assertFalse(tags.has("additionalProperties")); + assertEquals("string", tags.path("items").path("type").asText()); + assertFalse(tags.path("items").has("$comment")); + + JsonNode nested = cleaned.path("properties").path("nested"); + assertFalse(nested.has("$ref")); + assertEquals("number", nested.path("properties").path("inner").path("type").asText()); + } + + @Test + @DisplayName("drops non-string enum on a numeric type") + void dropsNumericEnum() { + JsonNode schema = parse(""" + { + "type": "object", + "properties": { + "duration": {"type": "integer", "enum": [60, 1440, 4320]}, + "mode": {"type": "string", "enum": ["fast", "slow"]} + } + } + """); + + ObjectNode cleaned = GeminiSchemaSanitizer.sanitizeToolParameters(schema, mapper); + + assertFalse(cleaned.path("properties").path("duration").has("enum"), + "integer enum with numeric literals must be dropped"); + assertTrue(cleaned.path("properties").path("mode").has("enum"), + "string enum is valid for Gemini and must be kept"); + } + + @Test + @DisplayName("null / empty input yields a minimal object schema") + void emptyInputYieldsObjectSchema() { + ObjectNode fromNull = GeminiSchemaSanitizer.sanitizeToolParameters(null, mapper); + assertEquals("object", fromNull.path("type").asText()); + assertTrue(fromNull.has("properties")); + + ObjectNode fromEmpty = GeminiSchemaSanitizer.sanitizeToolParameters(parse("{}"), mapper); + assertEquals("object", fromEmpty.path("type").asText()); + } +} diff --git a/mateclaw-ui/src/i18n/locales/en-US.ts b/mateclaw-ui/src/i18n/locales/en-US.ts index ae29ebb0..30b4b1fa 100644 --- a/mateclaw-ui/src/i18n/locales/en-US.ts +++ b/mateclaw-ui/src/i18n/locales/en-US.ts @@ -955,7 +955,7 @@ export default { openaiImageStatus: 'Reuses the OpenAI API Key configured in Model Management. No extra setup needed.', zhipuImageApiKey: 'Get from bigmodel.cn. CogView-3-Flash model is free. Shared with video generation.', falImageApiKey: 'Get from fal.ai for Flux image generation models. Shared with video generation.', - googleImagenInfo: 'Reuses Google API Key from Model Management. Supports Gemini image generation and Imagen 4.0.', + googleImagenInfo: 'Reuses the Gemini API Key from Model Management. Nano Banana Pro (Gemini 3 Pro Image) native generation — text-to-image and image editing.', minimaxImageInfo: 'Reuses MiniMax API Key from video settings. image-01 model, multiple aspect ratios, up to 9 images.', videoEnabled: 'Enable to let Agent use the video generation tool. Requires at least one configured video provider.', videoProvider: 'Select preferred video provider. Auto mode picks the first available one.', diff --git a/mateclaw-ui/src/i18n/locales/zh-CN.ts b/mateclaw-ui/src/i18n/locales/zh-CN.ts index 300329c4..14f0d3cb 100644 --- a/mateclaw-ui/src/i18n/locales/zh-CN.ts +++ b/mateclaw-ui/src/i18n/locales/zh-CN.ts @@ -846,7 +846,7 @@ export default { openaiImageStatus: '复用模型管理中配置的 OpenAI API Key,无需额外配置。', zhipuImageApiKey: '从 bigmodel.cn 获取。CogView-3-Flash 模型免费。与视频生成共用同一 Key。', falImageApiKey: '从 fal.ai 获取,支持 Flux 系列图片生成模型。与视频生成共用同一 Key。', - googleImagenInfo: '复用模型管理中的 Google API Key。支持 Gemini 图片生成和 Imagen 4.0 模型。', + googleImagenInfo: '复用模型管理中的 Gemini API Key。Nano Banana Pro(Gemini 3 Pro Image)原生图片生成,支持文生图与图片编辑。', minimaxImageInfo: '复用视频生成中的 MiniMax API Key。image-01 模型,支持多种画面比例,最多 9 张。', // 视频生成 videoEnabled: '开启后 Agent 可使用视频生成工具。需至少配置一个视频提供商的 API Key。', diff --git a/mateclaw-ui/src/views/Settings/Image/index.vue b/mateclaw-ui/src/views/Settings/Image/index.vue index 71ba8587..e2cea57d 100644 --- a/mateclaw-ui/src/views/Settings/Image/index.vue +++ b/mateclaw-ui/src/views/Settings/Image/index.vue @@ -33,7 +33,7 @@ - + @@ -145,10 +145,10 @@ - +

- Google Imagen + Google Gemini Image (Nano Banana) {{ t('settings.imageProviderTags.reuseLlmKey') }}