mirror of
https://gitee.com/mateos/mateclaw.git
synced 2026-09-13 11:13:43 +08:00
fix(llm): MiMo thinking 模式 reasoning_content 多轮对话兼容修复 (#189)
MiMo V2 系列在 thinking 模式下,assistant 消息携带 tool_calls 时必须同时包含 reasoning_content,否则提供方返回 400。 - ModelFamily 新增 MIMO_THINKING 族,detect() 添加 mimo* 匹配 - FallbackPolicy 新增 XIAOMI_MIMO(patchCrossTurn=true, patchNonToolCall=true) - 新增 ReasoningContentCache,按 tool_call_ids 回放真实推理内容 - 缓存作用范围:所有 patchCrossTurn=true 的 thinking provider(MiMo + DeepSeek) - NodeStreamingChatHelper 流式响应完成后写入缓存 Closes #188
This commit is contained in:
parent
02407871f4
commit
7861f603eb
@ -11,6 +11,7 @@ import org.springframework.ai.chat.prompt.Prompt;
|
||||
import org.springframework.web.reactive.function.client.WebClientResponseException;
|
||||
import vip.mate.channel.web.ChatStreamTracker;
|
||||
import vip.mate.llm.chatmodel.AssistantThinkingRelay;
|
||||
import vip.mate.llm.chatmodel.ReasoningContentCache;
|
||||
|
||||
import reactor.core.Disposable;
|
||||
|
||||
@ -1206,6 +1207,10 @@ public class NodeStreamingChatHelper {
|
||||
|
||||
AssistantMessage assembledMessage = buildAssistantMessageWithThinking(fullContent, fullThinking, finalToolCalls);
|
||||
|
||||
// Cache reasoning_content for MiMo-style providers that require it on
|
||||
// subsequent turns.
|
||||
cacheReasoningContent(fullThinking, finalToolCalls);
|
||||
|
||||
recordCacheMetrics(phase, promptTok, completionTok, cacheReadTok, cacheWriteTok);
|
||||
return new StreamResult(fullContent, fullThinking, assembledMessage,
|
||||
finalToolCalls, !finalToolCalls.isEmpty(), promptTok, completionTok,
|
||||
@ -1235,6 +1240,10 @@ public class NodeStreamingChatHelper {
|
||||
|
||||
AssistantMessage assembledMessage = buildAssistantMessageWithThinking(fullContent, fullThinking, finalToolCalls);
|
||||
|
||||
// Cache reasoning_content for MiMo-style providers that require it on
|
||||
// subsequent turns. The cache replays real values instead of empty strings.
|
||||
cacheReasoningContent(fullThinking, finalToolCalls);
|
||||
|
||||
recordCacheMetrics(phase, promptTok, completionTok, cacheReadTok, cacheWriteTok);
|
||||
return new StreamResult(fullContent, fullThinking, assembledMessage,
|
||||
finalToolCalls, !finalToolCalls.isEmpty(), promptTok, completionTok,
|
||||
@ -1268,6 +1277,24 @@ public class NodeStreamingChatHelper {
|
||||
return builder.build();
|
||||
}
|
||||
|
||||
/**
|
||||
* Store reasoning content in the cache for cross-turn replay.
|
||||
* Only caches when there are tool calls (MiMo requires reasoning_content
|
||||
* specifically on assistant messages with tool_calls).
|
||||
*/
|
||||
private static void cacheReasoningContent(String fullThinking,
|
||||
List<AssistantMessage.ToolCall> toolCalls) {
|
||||
if (fullThinking == null || fullThinking.isBlank()) return;
|
||||
if (toolCalls == null || toolCalls.isEmpty()) return;
|
||||
List<String> ids = toolCalls.stream()
|
||||
.map(AssistantMessage.ToolCall::id)
|
||||
.filter(id -> id != null && !id.isEmpty())
|
||||
.toList();
|
||||
if (!ids.isEmpty()) {
|
||||
ReasoningContentCache.store(ids, fullThinking);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Record token / cache usage to the optional metrics aggregator.
|
||||
* Called only from successful assembly paths ({@link #assembleResult}
|
||||
|
||||
@ -137,12 +137,17 @@ final class OpenAiRequestRewriter {
|
||||
if (next != null && !next.isEmpty()) {
|
||||
injected = next;
|
||||
} else {
|
||||
injected = policy.emptyFallback;
|
||||
if (injected == null && policy.warnOnMissingReal) {
|
||||
log.warn("[patchReasoningContent] provider={} requires real reasoning_content "
|
||||
+ "but relay has no value for assistant message at index {}; "
|
||||
+ "leaving null so provider returns explicit error.",
|
||||
providerIdOrUnknown(provider), i);
|
||||
// For cross-turn messages, try the reasoning content cache
|
||||
// (real values from prior responses) before falling back to empty.
|
||||
injected = resolveCrossTurnReasoning(msg, i <= lastUserIdx);
|
||||
if (injected == null) {
|
||||
injected = policy.emptyFallback;
|
||||
if (injected == null && policy.warnOnMissingReal) {
|
||||
log.warn("[patchReasoningContent] provider={} requires real reasoning_content "
|
||||
+ "but relay has no value for assistant message at index {}; "
|
||||
+ "leaving null so provider returns explicit error.",
|
||||
providerIdOrUnknown(provider), i);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (injected == null && msg.reasoningContent() == null) {
|
||||
@ -225,10 +230,11 @@ final class OpenAiRequestRewriter {
|
||||
* OpenAI-compatible gateway) might still require the patch.
|
||||
*/
|
||||
private enum FallbackPolicy {
|
||||
DEEPSEEK(" ", false, true, true),
|
||||
KIMI (" ", false, false, false),
|
||||
OPENAI (" ", false, false, false),
|
||||
DEFAULT (" ", false, false, false);
|
||||
DEEPSEEK (" ", false, true, true),
|
||||
KIMI (" ", false, false, false),
|
||||
OPENAI (" ", false, false, false),
|
||||
XIAOMI_MIMO (" ", false, true, true),
|
||||
DEFAULT (" ", false, false, false);
|
||||
|
||||
final String emptyFallback;
|
||||
final boolean warnOnMissingReal;
|
||||
@ -253,6 +259,7 @@ final class OpenAiRequestRewriter {
|
||||
case "deepseek" -> DEEPSEEK;
|
||||
case "kimi-cn", "kimi-intl", "kimi-code" -> KIMI;
|
||||
case "openai", "azure-openai" -> OPENAI;
|
||||
case "xiaomi-mimo" -> XIAOMI_MIMO;
|
||||
default -> DEFAULT;
|
||||
};
|
||||
}
|
||||
@ -306,6 +313,22 @@ final class OpenAiRequestRewriter {
|
||||
return family.isThinking();
|
||||
}
|
||||
|
||||
/**
|
||||
* Look up cached reasoning content for cross-turn assistant messages.
|
||||
* Returns the cached value, or {@code null} if no cache hit (caller falls
|
||||
* back to the policy's empty fallback).
|
||||
*/
|
||||
private static String resolveCrossTurnReasoning(
|
||||
OpenAiApi.ChatCompletionMessage msg, boolean isCrossTurn) {
|
||||
if (!isCrossTurn) return null;
|
||||
if (msg.toolCalls() == null || msg.toolCalls().isEmpty()) return null;
|
||||
List<String> ids = msg.toolCalls().stream()
|
||||
.map(tc -> tc.id())
|
||||
.filter(id -> id != null && !id.isEmpty())
|
||||
.toList();
|
||||
return ReasoningContentCache.get(ids);
|
||||
}
|
||||
|
||||
// ==================== reasoning_effort sanitizing ====================
|
||||
|
||||
/**
|
||||
|
||||
@ -0,0 +1,116 @@
|
||||
package vip.mate.llm.chatmodel;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
|
||||
/**
|
||||
* Static singleton cache for MiMo-style {@code reasoning_content} replay.
|
||||
*
|
||||
* <p>MiMo (and similar providers) require {@code reasoning_content} on assistant
|
||||
* messages that carry {@code tool_calls}. When the conversation spans multiple
|
||||
* turns, the cache replays the <em>real</em> reasoning content from prior
|
||||
* responses instead of injecting empty strings, preserving model context.
|
||||
*
|
||||
* <h2>Key design</h2>
|
||||
* <ul>
|
||||
* <li>Key: sorted, concatenated tool_call IDs from the assistant message.
|
||||
* Tool call IDs are unique per response, so this key naturally
|
||||
* disambiguates across turns.</li>
|
||||
* <li>TTL: 24 hours (configurable). Entries older than TTL are lazily evicted
|
||||
* on access and periodically during {@link #store}.</li>
|
||||
* <li>Max entries: 10,000. Oldest entries evicted when exceeded.</li>
|
||||
* </ul>
|
||||
*
|
||||
* <h2>Usage</h2>
|
||||
* <ol>
|
||||
* <li><b>Store</b>: after a streaming response completes, call
|
||||
* {@link #store} with the tool_call IDs and reasoning content.</li>
|
||||
* <li><b>Retrieve</b>: during request patching, call {@link #get} for
|
||||
* cross-turn assistant messages to fill in cached reasoning.</li>
|
||||
* </ol>
|
||||
*
|
||||
* @author MateClaw Team
|
||||
*/
|
||||
@Slf4j
|
||||
public final class ReasoningContentCache {
|
||||
|
||||
private static final long DEFAULT_MAX_AGE_MS = 24 * 60 * 60 * 1000L; // 24 hours
|
||||
private static final int DEFAULT_MAX_ENTRIES = 10_000;
|
||||
private static final long EVICT_INTERVAL_MS = 5 * 60 * 1000L; // 5 minutes
|
||||
|
||||
private static final ConcurrentHashMap<String, Entry> MAP = new ConcurrentHashMap<>();
|
||||
private static volatile long lastEvictMs = System.currentTimeMillis();
|
||||
|
||||
private ReasoningContentCache() {}
|
||||
|
||||
/**
|
||||
* Cache reasoning content for a set of tool_call IDs.
|
||||
*
|
||||
* @param toolCallIds tool call IDs from the assistant message (must not be null/empty)
|
||||
* @param reasoningContent the real reasoning content to cache (must not be blank)
|
||||
*/
|
||||
public static void store(List<String> toolCallIds, String reasoningContent) {
|
||||
if (toolCallIds == null || toolCallIds.isEmpty()) return;
|
||||
if (reasoningContent == null || reasoningContent.isBlank()) return;
|
||||
|
||||
String key = makeKey(toolCallIds);
|
||||
MAP.put(key, new Entry(reasoningContent, System.currentTimeMillis()));
|
||||
|
||||
maybeEvict();
|
||||
}
|
||||
|
||||
/**
|
||||
* Retrieve cached reasoning content for the given tool_call IDs.
|
||||
*
|
||||
* @return cached reasoning content, or {@code null} if not found or expired
|
||||
*/
|
||||
public static String get(List<String> toolCallIds) {
|
||||
if (toolCallIds == null || toolCallIds.isEmpty()) return null;
|
||||
|
||||
String key = makeKey(toolCallIds);
|
||||
Entry entry = MAP.get(key);
|
||||
if (entry == null) return null;
|
||||
|
||||
if (System.currentTimeMillis() - entry.storedAtMs > DEFAULT_MAX_AGE_MS) {
|
||||
MAP.remove(key);
|
||||
return null;
|
||||
}
|
||||
return entry.reasoningContent;
|
||||
}
|
||||
|
||||
/** Clear all cached entries. */
|
||||
public static void clear() {
|
||||
MAP.clear();
|
||||
}
|
||||
|
||||
/** Current cache size (for diagnostics). */
|
||||
public static int size() {
|
||||
return MAP.size();
|
||||
}
|
||||
|
||||
private static String makeKey(List<String> toolCallIds) {
|
||||
return String.join("|", toolCallIds.stream().sorted().toList());
|
||||
}
|
||||
|
||||
private static void maybeEvict() {
|
||||
long now = System.currentTimeMillis();
|
||||
if (now - lastEvictMs < EVICT_INTERVAL_MS) return;
|
||||
lastEvictMs = now;
|
||||
|
||||
// Remove expired entries
|
||||
MAP.entrySet().removeIf(e -> now - e.getValue().storedAtMs > DEFAULT_MAX_AGE_MS);
|
||||
|
||||
// Remove oldest if over limit
|
||||
if (MAP.size() > DEFAULT_MAX_ENTRIES) {
|
||||
MAP.entrySet().stream()
|
||||
.sorted((a, b) -> Long.compare(a.getValue().storedAtMs, b.getValue().storedAtMs))
|
||||
.limit(MAP.size() - DEFAULT_MAX_ENTRIES)
|
||||
.forEach(e -> MAP.remove(e.getKey()));
|
||||
}
|
||||
}
|
||||
|
||||
private record Entry(String reasoningContent, long storedAtMs) {}
|
||||
}
|
||||
@ -58,6 +58,15 @@ public enum ModelFamily {
|
||||
*/
|
||||
DEEPSEEK_V4_REASONING(false, false, true, false, false, true),
|
||||
|
||||
/**
|
||||
* Xiaomi MiMo thinking 模型:MiMo-VL-*、mimo-* 系列。
|
||||
* <p>
|
||||
* MiMo 的 thinking 模式与 DeepSeek 类似:响应返回 {@code reasoning_content},
|
||||
* 后续多轮请求必须将 {@code reasoning_content} 传回,否则 API 返回 400 错误。
|
||||
* 约束:保留 max_tokens;不支持 reasoning_effort;temperature/topP 用配置值。
|
||||
*/
|
||||
MIMO_THINKING(false, false, false, false, false, true),
|
||||
|
||||
/**
|
||||
* 通用 thinking 模型(名称含 "thinking" 或 "reasoner" 但不匹配上述族):
|
||||
* 如 qwen3-235b-a22b-thinking-2507
|
||||
@ -160,6 +169,11 @@ public enum ModelFamily {
|
||||
return DEEPSEEK_REASONER;
|
||||
}
|
||||
|
||||
// Xiaomi MiMo thinking 族:mimo-* / MiMo-VL-* 系列
|
||||
if (normalized.startsWith("mimo")) {
|
||||
return MIMO_THINKING;
|
||||
}
|
||||
|
||||
// 通用 thinking 族:名称含 thinking / reasoner 关键词
|
||||
if (normalized.contains("thinking") || normalized.contains("reasoner")) {
|
||||
return GENERIC_THINKING;
|
||||
|
||||
@ -0,0 +1,69 @@
|
||||
package vip.mate.llm.chatmodel;
|
||||
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.DisplayName;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
|
||||
class ReasoningContentCacheTest {
|
||||
|
||||
@AfterEach
|
||||
void cleanup() {
|
||||
ReasoningContentCache.clear();
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("Store and retrieve reasoning content by tool_call IDs")
|
||||
void storeAndGet() {
|
||||
List<String> ids = List.of("call_1", "call_2");
|
||||
ReasoningContentCache.store(ids, "thinking content here");
|
||||
|
||||
assertEquals("thinking content here", ReasoningContentCache.get(ids));
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("Cache key is order-independent (sorted tool_call IDs)")
|
||||
void orderIndependent() {
|
||||
ReasoningContentCache.store(List.of("call_b", "call_a"), "content");
|
||||
|
||||
assertEquals("content", ReasoningContentCache.get(List.of("call_a", "call_b")));
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("Miss returns null")
|
||||
void cacheMiss() {
|
||||
assertNull(ReasoningContentCache.get(List.of("nonexistent")));
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("Empty/null tool_call IDs are no-ops")
|
||||
void emptyIds() {
|
||||
ReasoningContentCache.store(List.of(), "content");
|
||||
ReasoningContentCache.store(null, "content");
|
||||
assertEquals(0, ReasoningContentCache.size());
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("Blank/null reasoning content is not cached")
|
||||
void blankContent() {
|
||||
ReasoningContentCache.store(List.of("call_1"), "");
|
||||
ReasoningContentCache.store(List.of("call_1"), " ");
|
||||
ReasoningContentCache.store(List.of("call_1"), null);
|
||||
assertEquals(0, ReasoningContentCache.size());
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("Clear removes all entries")
|
||||
void clearAll() {
|
||||
ReasoningContentCache.store(List.of("call_1"), "content1");
|
||||
ReasoningContentCache.store(List.of("call_2"), "content2");
|
||||
assertEquals(2, ReasoningContentCache.size());
|
||||
|
||||
ReasoningContentCache.clear();
|
||||
assertEquals(0, ReasoningContentCache.size());
|
||||
assertNull(ReasoningContentCache.get(List.of("call_1")));
|
||||
}
|
||||
}
|
||||
@ -65,4 +65,15 @@ class ModelFamilyTest {
|
||||
assertEquals(ModelFamily.STANDARD, ModelFamily.detect(""));
|
||||
assertEquals(ModelFamily.STANDARD, ModelFamily.detect(" "));
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("Xiaomi MiMo models → MIMO_THINKING (reasoning_content relay required)")
|
||||
void mimo_thinking() {
|
||||
assertEquals(ModelFamily.MIMO_THINKING, ModelFamily.detect("mimo-v2-flash"));
|
||||
assertEquals(ModelFamily.MIMO_THINKING, ModelFamily.detect("MiMo-VL-7B-RL"));
|
||||
assertTrue(ModelFamily.MIMO_THINKING.isThinking(),
|
||||
"Mimo must be flagged as thinking so reasoning_content is patched");
|
||||
assertFalse(ModelFamily.MIMO_THINKING.supportsReasoningEffort(),
|
||||
"Mimo does not accept the reasoning_effort parameter");
|
||||
}
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user